CHANGELOG.txt

@@ -393,16 +393,3 @@ v0.6.22
 
 - Add background scraping jobs for Bookmarks and logged-in user's User Feed
 - Manually rescraping a User Feed now re-downloads user's profile and banner images as well
-
-v0.7.0
-------
-
-- PKG: Type definitions are now in `pkg/persistence` instead of `pkg/scraper`
-- BUGFIX: fix notifications page not having user's "likes" filled out on tweets
-- Compound notifications now tell you how many tweets were "liked", how many users followed you, etc
-
-v0.7.1
-------
-
-- BUGFIX: fix banned users getting their user info (followers count, profile image, display name) getting erased
-- PKG: webserver is now a public package

cmd/compile.sh

@@ -4,7 +4,7 @@ set -x
 set -e
 
 # General build flags
-FLAGS="-s -w -X gitlab.com/offline-twitter/twitter_offline_engine/pkg/webserver.use_embedded=true"
+FLAGS="-s -w -X gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver.use_embedded=true"
 
 # Check for the `--static` flag and consume it
 USE_STATIC=false

cmd/tests.sh

@@ -349,11 +349,6 @@ test $(sqlite3 twitter.db "select is_stub, user_id = 0x4000000000000000 from twe
 tw fetch_user nancytracker
 test "$(sqlite3 twitter.db "select is_banned from users where handle='nancytracker'")" = "1"
 
-# Fetch a thread with a banned user in it; test no-clobbering of the existing user data
-sqlite3 twitter.db "insert into users (id, handle, display_name, profile_image_url, banner_image_local_path, followers_count) values (1595500307374829568, 'spandrell3', 'Spandrell', 'Profile URL', 'Banner Local', 10)"
-tw fetch_tweet https://twitter.com/spandrell3/status/1709580398026805736
-test "$(sqlite3 twitter.db "select display_name, profile_image_url, banner_image_local_path, followers_count, is_banned from users where handle='spandrell3'")" = "Spandrell|Profile URL|Banner Local|10|1"
-
 
 # Fetch a user with "600x200" banner image
 tw fetch_user AlexKoppelman  # This is probably kind of a flimsy test

cmd/twitter/helpers.go

@@ -112,9 +112,6 @@ func full_save_tweet_trove(trove TweetTrove) {
 			// Mark them as deleted.
 			// Handle and display name won't be updated if the user exists.
 			updated_user = User{ID: u_id, DisplayName: "<Unknown User>", Handle: "<UNKNOWN USER>", IsDeleted: true}
-		} else if errors.Is(err, scraper.ErrUserIsBanned) {
-			// Mark them as banned (also won't clobber handle and display name)
-			updated_user = User{ID: u_id, DisplayName: "<Unknown User>", Handle: "<UNKNOWN USER>", IsBanned: true}
 		} else if err != nil {
 			panic(fmt.Errorf("error scraping conflicting user (ID %d): %w", u_id, err))
 		}

cmd/twitter/main.go

@@ -14,9 +14,9 @@ import (
 	"syscall"
 	"time"
 
+	"gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver"
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
 	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
-	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/webserver"
 )
 
 // Global variable referencing the open data profile

cmd/windows-compile.sh

@@ -14,12 +14,12 @@ if [[ -z "$1" ]]; then
 fi
 
 # Always use static build for windows
-FLAGS="-s -w -X gitlab.com/offline-twitter/twitter_offline_engine/pkg/webserver.use_embedded=true"
+FLAGS="-s -w -X gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver.use_embedded=true"
 SPECIAL_FLAGS_FOR_STATIC_BUILD="-linkmode=external -extldflags=-static"
 
 # Add an application icon using `windres` and `.syso`
 # The `.syso` should be picked up automatically, since it's in the same directory we run `go build` from`
-echo '1 ICON "../pkg/webserver/static/twitter.ico"' > /tmp/icon.rc
+echo '1 ICON "../internal/webserver/static/twitter.ico"' > /tmp/icon.rc
 x86_64-w64-mingw32-windres /tmp/icon.rc -o icon.syso
 
 go build -ldflags="$FLAGS $SPECIAL_FLAGS_FOR_STATIC_BUILD -X main.version_string=$1" -o twitter.exe ./twitter

doc/TODO.txt

@@ -136,6 +136,9 @@ TOOD: notifications-duplicate-bug (#scraper, #bug)
 TODO: dm-reaccs-popup-list (#dms, #webserver, #reaccs)
 - on-click popup showing who sent which reacc
 
+TODO: detect-banned-users-when-scraping-tweets (#scraper, #bug)
+- banned users aren't detected when scraping tweets
+
 TODO: sqlite-query-optimizing
 - https://sqlite.org/cli.html#index_recommendations_sqlite_expert_
 - https://sqlite.org/eqp.html
@@ -147,8 +150,3 @@ TODO: deprecated-offline-follows
 - Offline Follows doesn't do much userful anymore
 
 TODO: offline-timeline-vs-user-timeline
-
-TODO: go-stdlib-path-vs-filepath (#code-quality)
-
-TODO: show-retweet-icon-in-ui (#webserver)
-- The UI should show whether the active user has retweeted the tweet, same as with "likes"

doc/tasks/bash-completions.todo

@@ -1,9 +1,3 @@
 Bash completions
 
 https://opensource.com/article/18/3/creating-bash-completion-script
-
-
-Enable with:
-```
-complete -C 'go run ./twitter autocomplete "$@"' twitter
-```

doc/tasks/notifications-mark-as-read.todo

@@ -0,0 +1,8 @@
+Notifications mark as read
+
+- it uses cursors.  so, to mark notifications as read:
+	- scrape notifications
+	- extract top cursor
+	- use that as the cursor value to mark as read
+
+DAABDAABCgABFDzeDIfVUAIIAAIAAAABCAADiJONcggABOxI6DUACwACAAAAC0FaSHRBaThqLUN3CAADyS991gAA

internal/webserver/handler_messages_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/html"
 
+	"gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver"
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
-	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/webserver"
 )
 
 func TestMessagesIndexPageRequiresActiveUser(t *testing.T) {

internal/webserver/server_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/andybalholm/cascadia"
 	"github.com/stretchr/testify/require"
 
+	"gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver"
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
-	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/webserver"
 )
 
 type CapturingWriter struct {

internal/webserver/temp_utils.go

@@ -20,11 +20,8 @@ func (app *Application) full_save_tweet_trove(trove TweetTrove) {
 		updated_user, err := scraper.GetUserByID(u_id)
 		if errors.Is(err, scraper.ErrDoesntExist) {
 			// Mark them as deleted.
-			// Handle and display name won't be clobbered if the user exists.
+			// Handle and display name won't be updated if the user exists.
 			updated_user = User{ID: u_id, DisplayName: "<Unknown User>", Handle: "<UNKNOWN USER>", IsDeleted: true}
-		} else if errors.Is(err, scraper.ErrUserIsBanned) {
-			// Mark them as banned (also won't clobber handle and display name)
-			updated_user = User{ID: u_id, DisplayName: "<Unknown User>", Handle: "<UNKNOWN USER>", IsBanned: true}
 		} else if err != nil {
 			panic(fmt.Errorf("error scraping conflicting user (ID %d): %w", u_id, err))
 		}