me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: apply dependency injection change (whole API object -> just the downloader function) to 'SaveTweetTrove' method

Browse Source
This commit is contained in:
Alessio 2025-02-03 21:58:53 -08:00
parent 0c1d853f55
commit cb0b478c08
9 changed files with 44 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cmd/twitter/main.go
View File

@ -410,7 +410,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(err.Error(), false, -1) die(err.Error(), false, -1)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
} }
@ -431,7 +431,7 @@ func fetch_user_feed(handle string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
@ -449,7 +449,7 @@ func get_user_likes(handle string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
@ -467,7 +467,7 @@ func get_followees(handle string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2) die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
profile.SaveAsFolloweesList(user.ID, trove) profile.SaveAsFolloweesList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err) happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err)
@ -481,7 +481,7 @@ func get_followers(handle string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2) die(fmt.Sprintf("Error getting followees: %s\n %s", handle, err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
profile.SaveAsFollowersList(user.ID, trove) profile.SaveAsFollowersList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), err) happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), err)
@ -491,7 +491,7 @@ func get_bookmarks(how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error scraping bookmarks:\n %s", err.Error()), false, -2) die(fmt.Sprintf("Error scraping bookmarks:\n %s", err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf( happy_exit(fmt.Sprintf(
"Saved %d tweets, %d retweets, %d users, and %d bookmarks", "Saved %d tweets, %d retweets, %d users, and %d bookmarks",
@ -504,7 +504,7 @@ func fetch_timeline(is_following_only bool) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2) die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
@ -544,7 +544,7 @@ func search(query string, how_many int) {
if is_scrape_failure(err) { if is_scrape_failure(err) {
die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100) die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
} }
@ -607,7 +607,7 @@ func fetch_inbox(how_many int) {
if err != nil { if err != nil {
die(fmt.Sprintf("Failed to fetch inbox:\n %s", err.Error()), false, 1) die(fmt.Sprintf("Failed to fetch inbox:\n %s", err.Error()), false, 1)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
} }
@ -621,7 +621,7 @@ func fetch_dm(id string, how_many int) {
if err != nil { if err != nil {
die(fmt.Sprintf("Failed to fetch dm:\n %s", err.Error()), false, 1) die(fmt.Sprintf("Failed to fetch dm:\n %s", err.Error()), false, 1)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit( happy_exit(
fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)),
err, err,
@ -638,7 +638,7 @@ func send_dm(room_id string, text string, in_reply_to_id int) {
if err != nil { if err != nil {
die(fmt.Sprintf("Failed to send dm:\n %s", err.Error()), false, 1) die(fmt.Sprintf("Failed to send dm:\n %s", err.Error()), false, 1)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
} }
@ -670,7 +670,7 @@ func get_notifications(how_many int) {
panic(err) panic(err)
} }
profile.SaveTweetTrove(trove, true, &api) profile.SaveTweetTrove(trove, true, api.DownloadMedia)
happy_exit(fmt.Sprintf("Saved %d notifications, %d tweets and %d users", happy_exit(fmt.Sprintf("Saved %d notifications, %d tweets and %d users",
len(trove.Notifications), len(trove.Tweets), len(trove.Users), len(trove.Notifications), len(trove.Tweets), len(trove.Users),
), nil) ), nil)

4
internal/webserver/handler_bookmarks.go
View File

@ -26,8 +26,8 @@ func (app *Application) Bookmarks(w http.ResponseWriter, r *http.Request) {
panic(err) // Return a toast panic(err) // Return a toast
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
c := persistence.NewUserFeedBookmarksCursor(app.ActiveUser.Handle) c := persistence.NewUserFeedBookmarksCursor(app.ActiveUser.Handle)

12
internal/webserver/handler_login.go
View File

@ -89,8 +89,8 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api
http.Redirect(w, r, "/", 303) http.Redirect(w, r, "/", 303)
} }
fmt.Println("Saving initial feed results...") fmt.Println("Saving initial feed results...")
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
// Scrape the user's followers // Scrape the user's followers
trove, err = app.API.GetFollowees(user.ID, 1000) trove, err = app.API.GetFollowees(user.ID, 1000)
@ -98,9 +98,9 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api
app.ErrorLog.Printf("Failed to scrape followers: %s", err.Error()) app.ErrorLog.Printf("Failed to scrape followers: %s", err.Error())
http.Redirect(w, r, "/", 303) http.Redirect(w, r, "/", 303)
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
app.Profile.SaveAsFolloweesList(user.ID, trove) app.Profile.SaveAsFolloweesList(user.ID, trove)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
// Redirect to Timeline // Redirect to Timeline
http.Redirect(w, r, "/", 303) http.Redirect(w, r, "/", 303)
@ -129,8 +129,8 @@ func (app *Application) ChangeSession(w http.ResponseWriter, r *http.Request) {
return return
} }
// We have to save the notifications first, otherwise it'll just report 0 since the last-read sort index // We have to save the notifications first, otherwise it'll just report 0 since the last-read sort index
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
// Set the notifications count // Set the notifications count
app.LastReadNotificationSortIndex = last_unread_notification_sort_index app.LastReadNotificationSortIndex = last_unread_notification_sort_index
}() }()

12
internal/webserver/handler_messages.go
View File

@ -80,8 +80,8 @@ func (app *Application) message_send(w http.ResponseWriter, r *http.Request) {
if err != nil { if err != nil {
panic(err) panic(err)
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) {
@ -153,8 +153,8 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) {
if err != nil { if err != nil {
panic(err) panic(err)
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) // Download the content in the background go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Download the content in the background
} }
// `LatestPollingTimestamp` sort of passes-through the function; if we're not updating it, it // `LatestPollingTimestamp` sort of passes-through the function; if we're not updating it, it
@ -244,8 +244,8 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) {
panic(err) panic(err)
} }
inbox_cursor = new_cursor inbox_cursor = new_cursor
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")

4
internal/webserver/handler_search.go
View File

@ -107,8 +107,8 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) {
app.ErrorLog.Print(err) app.ErrorLog.Print(err)
// TOOD: show error in UI // TOOD: show error in UI
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
c, err := persistence.NewCursorFromSearchQuery(search_text) c, err := persistence.NewCursorFromSearchQuery(search_text)

4
internal/webserver/handler_tweet_detail.go
View File

@ -53,8 +53,8 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv
// Save the trove unless there was an unrecoverable error // Save the trove unless there was an unrecoverable error
if err == nil || errors.Is(err, scraper.END_OF_FEED) || errors.Is(err, scraper.ErrRateLimited) { if err == nil || errors.Is(err, scraper.END_OF_FEED) || errors.Is(err, scraper.ErrRateLimited) {
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) // Download the content in the background go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia) // Download the content in the background
_, is_available = trove.Tweets[id] _, is_available = trove.Tweets[id]
} }

16
internal/webserver/handler_user_feed.go
View File

@ -59,16 +59,16 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
app.ErrorLog.Print(err) app.ErrorLog.Print(err)
// TOOD: show error in UI // TOOD: show error in UI
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} else if len(parts) == 2 && parts[1] == "likes" { } else if len(parts) == 2 && parts[1] == "likes" {
trove, err := app.API.GetUserLikes(user.ID, 50) // TODO: parameterizable trove, err := app.API.GetUserLikes(user.ID, 50) // TODO: parameterizable
if err != nil { if err != nil {
app.ErrorLog.Print(err) app.ErrorLog.Print(err)
// TOOD: show error in UI // TOOD: show error in UI
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
} }
@ -171,9 +171,9 @@ func (app *Application) UserFollowees(w http.ResponseWriter, r *http.Request, us
app.ErrorLog.Print(err) app.ErrorLog.Print(err)
// TOOD: show error in UI // TOOD: show error in UI
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
app.Profile.SaveAsFolloweesList(user.ID, trove) app.Profile.SaveAsFolloweesList(user.ID, trove)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
data, trove := NewFollowsData(app.Profile.GetFollowees(user.ID)) data, trove := NewFollowsData(app.Profile.GetFollowees(user.ID))
@ -197,9 +197,9 @@ func (app *Application) UserFollowers(w http.ResponseWriter, r *http.Request, us
app.ErrorLog.Print(err) app.ErrorLog.Print(err)
// TOOD: show error in UI // TOOD: show error in UI
} }
app.Profile.SaveTweetTrove(trove, false, &app.API) app.Profile.SaveTweetTrove(trove, false, app.API.DownloadMedia)
app.Profile.SaveAsFollowersList(user.ID, trove) app.Profile.SaveAsFollowersList(user.ID, trove)
go app.Profile.SaveTweetTrove(trove, true, &app.API) go app.Profile.SaveTweetTrove(trove, true, app.API.DownloadMedia)
} }
data, trove := NewFollowsData(app.Profile.GetFollowers(user.ID)) data, trove := NewFollowsData(app.Profile.GetFollowers(user.ID))

4
internal/webserver/stopwatch.go
View File

@ -49,8 +49,8 @@ func (t *BackgroundTask) Do() {
// Run the task // Run the task
trove := t.GetTroveFunc(&t.app.API) trove := t.GetTroveFunc(&t.app.API)
t.log.Print("saving results") t.log.Print("saving results")
t.app.Profile.SaveTweetTrove(trove, false, &t.app.API) t.app.Profile.SaveTweetTrove(trove, false, t.app.API.DownloadMedia)
go t.app.Profile.SaveTweetTrove(trove, true, &t.app.API) go t.app.Profile.SaveTweetTrove(trove, true, t.app.API.DownloadMedia)
t.log.Print("success") t.log.Print("success")
} }

8
pkg/persistence/tweet_trove_queries.go
View File

@ -12,7 +12,7 @@ import (
// Convenience function that saves all the objects in a TweetTrove. // Convenience function that saves all the objects in a TweetTrove.
// Panics if anything goes wrong. // Panics if anything goes wrong.
func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API) { func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, download func(string) ([]byte, error)) {
for i, u := range trove.Users { for i, u := range trove.Users {
err := p.SaveUser(&u) err := p.SaveUser(&u)
// Check for handle conflicts and handle them in place // Check for handle conflicts and handle them in place
@ -64,7 +64,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API
if should_download { if should_download {
// Download their tiny profile image // Download their tiny profile image
err = p.DownloadUserProfileImageTiny(&u, api.DownloadMedia) err = p.DownloadUserProfileImageTiny(&u, download)
if errors.Is(err, ErrRequestTimeout) { if errors.Is(err, ErrRequestTimeout) {
// Forget about it; if it's important someone will try again // Forget about it; if it's important someone will try again
fmt.Printf("Failed to @%s's tiny profile image (%q): %s\n", u.Handle, u.ProfileImageUrl, err.Error()) fmt.Printf("Failed to @%s's tiny profile image (%q): %s\n", u.Handle, u.ProfileImageUrl, err.Error())
@ -88,7 +88,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API
} }
if should_download { if should_download {
err = p.DownloadTweetContentFor(&t, api.DownloadMedia) err = p.DownloadTweetContentFor(&t, download)
if errors.Is(err, ErrRequestTimeout) || errors.Is(err, ErrMediaDownload404) { if errors.Is(err, ErrRequestTimeout) || errors.Is(err, ErrMediaDownload404) {
// Forget about it; if it's important someone will try again // Forget about it; if it's important someone will try again
fmt.Printf("Failed to download tweet ID %d: %s\n", t.ID, err.Error()) fmt.Printf("Failed to download tweet ID %d: %s\n", t.ID, err.Error())
@ -147,7 +147,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API
// Download content if needed // Download content if needed
if should_download { if should_download {
downloader := DefaultDownloader{Download: api.DownloadMedia} downloader := DefaultDownloader{Download: download}
for _, img := range m.Images { for _, img := range m.Images {
// Check if it's already downloaded // Check if it's already downloaded