From cb0b478c080f3ad147095231b9203ebf2c93589f Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 3 Feb 2025 21:58:53 -0800 Subject: [PATCH] REFACTOR: apply dependency injection change (whole API object -> just the downloader function) to 'SaveTweetTrove' method --- cmd/twitter/main.go | 24 +++++++++++----------- internal/webserver/handler_bookmarks.go | 4 ++-- internal/webserver/handler_login.go | 12 +++++------ internal/webserver/handler_messages.go | 12 +++++------ internal/webserver/handler_search.go | 4 ++-- internal/webserver/handler_tweet_detail.go | 4 ++-- internal/webserver/handler_user_feed.go | 16 +++++++-------- internal/webserver/stopwatch.go | 4 ++-- pkg/persistence/tweet_trove_queries.go | 8 ++++---- 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 9b69779..4c46ed9 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -410,7 +410,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) { if is_scrape_failure(err) { die(err.Error(), false, -1) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err) } @@ -431,7 +431,7 @@ func fetch_user_feed(handle string, how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit( fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), @@ -449,7 +449,7 @@ func get_user_likes(handle string, how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit( fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), @@ -467,7 +467,7 @@ func get_followees(handle string, how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) profile.SaveAsFolloweesList(user.ID, trove) happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err) @@ -481,7 +481,7 @@ func get_followers(handle string, how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) profile.SaveAsFollowersList(user.ID, trove) happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), err) @@ -491,7 +491,7 @@ func get_bookmarks(how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error scraping bookmarks:\n %s", err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf( "Saved %d tweets, %d retweets, %d users, and %d bookmarks", @@ -504,7 +504,7 @@ func fetch_timeline(is_following_only bool) { if is_scrape_failure(err) { die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit( fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), @@ -544,7 +544,7 @@ func search(query string, how_many int) { if is_scrape_failure(err) { die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err) } @@ -607,7 +607,7 @@ func fetch_inbox(how_many int) { if err != nil { die(fmt.Sprintf("Failed to fetch inbox:\n %s", err.Error()), false, 1) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil) } @@ -621,7 +621,7 @@ func fetch_dm(id string, how_many int) { if err != nil { die(fmt.Sprintf("Failed to fetch dm:\n %s", err.Error()), false, 1) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit( fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), err, @@ -638,7 +638,7 @@ func send_dm(room_id string, text string, in_reply_to_id int) { if err != nil { die(fmt.Sprintf("Failed to send dm:\n %s", err.Error()), false, 1) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil) } @@ -670,7 +670,7 @@ func get_notifications(how_many int) { panic(err) } - profile.SaveTweetTrove(trove, true, &api) + profile.SaveTweetTrove(trove, true, api.DownloadMedia) happy_exit(fmt.Sprintf("Saved %d notifications, %d tweets and %d users", len(trove.Notifications), len(trove.Tweets), len(trove.Users), ), nil) diff --git a/internal/webserver/handler_bookmarks.go b/internal/webserver/handler_bookmarks.go index d23069c..1afdd29 100644 --- a/internal/webserver/handler_bookmarks.go +++ b/internal/webserver/handler_bookmarks.go @@ -26,8 +26,8 @@ func (app *Application) Bookmarks(w http.ResponseWriter, r *http.Request) { panic(err) // Return a toast } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } c := persistence.NewUserFeedBookmarksCursor(app.ActiveUser.Handle) diff --git a/internal/webserver/handler_login.go b/internal/webserver/handler_login.go index 6169c1e..eb83bb5 100644 --- a/internal/webserver/handler_login.go +++ b/internal/webserver/handler_login.go @@ -89,8 +89,8 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api http.Redirect(w, r, "/", 303) } fmt.Println("Saving initial feed results...") - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Scrape the user's followers trove, err = app.API.GetFollowees(user.ID, 1000) @@ -98,9 +98,9 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api app.ErrorLog.Printf("Failed to scrape followers: %s", err.Error()) http.Redirect(w, r, "/", 303) } - app.Profile.SaveTweetTrove(trove, false, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) app.Profile.SaveAsFolloweesList(user.ID, trove) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Redirect to Timeline http.Redirect(w, r, "/", 303) @@ -129,8 +129,8 @@ func (app *Application) ChangeSession(w http.ResponseWriter, r *http.Request) { return } // We have to save the notifications first, otherwise it'll just report 0 since the last-read sort index - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Set the notifications count app.LastReadNotificationSortIndex = last_unread_notification_sort_index }() diff --git a/internal/webserver/handler_messages.go b/internal/webserver/handler_messages.go index 7ca4e38..c027e27 100644 --- a/internal/webserver/handler_messages.go +++ b/internal/webserver/handler_messages.go @@ -80,8 +80,8 @@ func (app *Application) message_send(w http.ResponseWriter, r *http.Request) { if err != nil { panic(err) } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { @@ -153,8 +153,8 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { if err != nil { panic(err) } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) // Download the content in the background + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Download the content in the background } // `LatestPollingTimestamp` sort of passes-through the function; if we're not updating it, it @@ -244,8 +244,8 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) { panic(err) } inbox_cursor = new_cursor - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") diff --git a/internal/webserver/handler_search.go b/internal/webserver/handler_search.go index 39371ea..ab53443 100644 --- a/internal/webserver/handler_search.go +++ b/internal/webserver/handler_search.go @@ -107,8 +107,8 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) { app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } c, err := persistence.NewCursorFromSearchQuery(search_text) diff --git a/internal/webserver/handler_tweet_detail.go b/internal/webserver/handler_tweet_detail.go index 1ad0f7a..7cb8a4b 100644 --- a/internal/webserver/handler_tweet_detail.go +++ b/internal/webserver/handler_tweet_detail.go @@ -53,8 +53,8 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv // Save the trove unless there was an unrecoverable error if err == nil || errors.Is(err, scraper.END_OF_FEED) || errors.Is(err, scraper.ErrRateLimited) { - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) // Download the content in the background + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Download the content in the background _, is_available = trove.Tweets[id] } diff --git a/internal/webserver/handler_user_feed.go b/internal/webserver/handler_user_feed.go index 6d343bc..bb200d1 100644 --- a/internal/webserver/handler_user_feed.go +++ b/internal/webserver/handler_user_feed.go @@ -59,16 +59,16 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } else if len(parts) == 2 && parts[1] == "likes" { trove, err := app.API.GetUserLikes(user.ID, 50) // TODO: parameterizable if err != nil { app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove, false, &app.API) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } } @@ -171,9 +171,9 @@ func (app *Application) UserFollowees(w http.ResponseWriter, r *http.Request, us app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove, false, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) app.Profile.SaveAsFolloweesList(user.ID, trove) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } data, trove := NewFollowsData(app.Profile.GetFollowees(user.ID)) @@ -197,9 +197,9 @@ func (app *Application) UserFollowers(w http.ResponseWriter, r *http.Request, us app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove, false, &app.API) + app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia) app.Profile.SaveAsFollowersList(user.ID, trove) - go app.Profile.SaveTweetTrove(trove, true, &app.API) + go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) } data, trove := NewFollowsData(app.Profile.GetFollowers(user.ID)) diff --git a/internal/webserver/stopwatch.go b/internal/webserver/stopwatch.go index 9b7c3ae..49b2c24 100644 --- a/internal/webserver/stopwatch.go +++ b/internal/webserver/stopwatch.go @@ -49,8 +49,8 @@ func (t *BackgroundTask) Do() { // Run the task trove := t.GetTroveFunc(&t.app.API) t.log.Print("saving results") - t.app.Profile.SaveTweetTrove(trove, false, &t.app.API) - go t.app.Profile.SaveTweetTrove(trove, true, &t.app.API) + t.app.Profile.SaveTweetTrove(trove, false, t.app.API.DownloadMedia) + go t.app.Profile.SaveTweetTrove(trove, true, t.app.API.DownloadMedia) t.log.Print("success") } diff --git a/pkg/persistence/tweet_trove_queries.go b/pkg/persistence/tweet_trove_queries.go index 01e958f..0dba9f2 100644 --- a/pkg/persistence/tweet_trove_queries.go +++ b/pkg/persistence/tweet_trove_queries.go @@ -12,7 +12,7 @@ import ( // Convenience function that saves all the objects in a TweetTrove. // Panics if anything goes wrong. -func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API) { +func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, download func(string) ([]byte, error)) { for i, u := range trove.Users { err := p.SaveUser(&u) // Check for handle conflicts and handle them in place @@ -64,7 +64,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API if should_download { // Download their tiny profile image - err = p.DownloadUserProfileImageTiny(&u, api.DownloadMedia) + err = p.DownloadUserProfileImageTiny(&u, download) if errors.Is(err, ErrRequestTimeout) { // Forget about it; if it's important someone will try again fmt.Printf("Failed to @%s's tiny profile image (%q): %s\n", u.Handle, u.ProfileImageUrl, err.Error()) @@ -88,7 +88,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API } if should_download { - err = p.DownloadTweetContentFor(&t, api.DownloadMedia) + err = p.DownloadTweetContentFor(&t, download) if errors.Is(err, ErrRequestTimeout) || errors.Is(err, ErrMediaDownload404) { // Forget about it; if it's important someone will try again fmt.Printf("Failed to download tweet ID %d: %s\n", t.ID, err.Error()) @@ -147,7 +147,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API // Download content if needed if should_download { - downloader := DefaultDownloader{Download: api.DownloadMedia} + downloader := DefaultDownloader{Download: download} for _, img := range m.Images { // Check if it's already downloaded