diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index cf92a09..b70b0cd 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -270,7 +270,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) { if err != nil { die(err.Error(), false, -1) } - profile.SaveTweetTrove(trove) + profile.SaveTweetTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users))) } @@ -291,7 +291,7 @@ func fetch_user_feed(handle string, how_many int) { if err != nil { die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove) + profile.SaveTweetTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } @@ -306,7 +306,7 @@ func get_user_likes(handle string, how_many int) { if err != nil { die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) } - profile.SaveTweetTrove(trove) + profile.SaveTweetTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } @@ -316,7 +316,7 @@ func fetch_timeline(is_for_you bool) { if err != nil { die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2) } - profile.SaveTweetTrove(trove) + profile.SaveTweetTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } @@ -353,7 +353,7 @@ func search(query string, how_many int) { if err != nil { die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100) } - profile.SaveTweetTrove(trove) + profile.SaveTweetTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users))) } @@ -413,7 +413,7 @@ func start_webserver(addr string) { func fetch_inbox(how_many int) { trove, _ := scraper.GetInbox(how_many) - profile.SaveDMTrove(trove) + profile.SaveDMTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms))) } @@ -424,7 +424,7 @@ func fetch_dm(id string, how_many int) { } max_id := scraper.DMMessageID(^uint(0) >> 1) trove := scraper.GetConversation(room.ID, max_id, how_many) - profile.SaveDMTrove(trove) + profile.SaveDMTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms))) } @@ -435,6 +435,6 @@ func send_dm(room_id string, text string, in_reply_to_id int) { } trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id)) - profile.SaveDMTrove(trove) + profile.SaveDMTrove(trove, true) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms))) } diff --git a/internal/webserver/handler_tweet_detail.go b/internal/webserver/handler_tweet_detail.go index cb24379..21e1c6f 100644 --- a/internal/webserver/handler_tweet_detail.go +++ b/internal/webserver/handler_tweet_detail.go @@ -65,7 +65,8 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv if is_needing_scrape && !app.IsScrapingDisabled { trove, err := scraper.GetTweetFullAPIV2(id, 50) // TODO: parameterizable if err == nil { - app.Profile.SaveTweetTrove(trove) + app.Profile.SaveTweetTrove(trove, false) + go app.Profile.SaveTweetTrove(trove, true) // Download the content in the background is_available = true } else { app.ErrorLog.Print(err) diff --git a/internal/webserver/handler_user_feed.go b/internal/webserver/handler_user_feed.go index 6f309fe..a4b8c58 100644 --- a/internal/webserver/handler_user_feed.go +++ b/internal/webserver/handler_user_feed.go @@ -63,14 +63,16 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove) + app.Profile.SaveTweetTrove(trove, false) + go app.Profile.SaveTweetTrove(trove, true) } else if len(parts) == 2 && parts[1] == "likes" { trove, err := scraper.GetUserLikes(user.ID, 50) // TODO: parameterizable if err != nil { app.ErrorLog.Print(err) // TOOD: show error in UI } - app.Profile.SaveTweetTrove(trove) + app.Profile.SaveTweetTrove(trove, false) + go app.Profile.SaveTweetTrove(trove, true) } } diff --git a/internal/webserver/stopwatch.go b/internal/webserver/stopwatch.go index c2a7e8d..499b2fb 100644 --- a/internal/webserver/stopwatch.go +++ b/internal/webserver/stopwatch.go @@ -37,7 +37,8 @@ func (app *Application) background_scrape() { return } fmt.Println("Saving scrape results...") - app.Profile.SaveTweetTrove(trove) + app.Profile.SaveTweetTrove(trove, false) + go app.Profile.SaveTweetTrove(trove, true) fmt.Println("Scraping succeeded.") is_for_you_only = false } @@ -71,7 +72,8 @@ func (app *Application) background_user_likes_scrape() { return } fmt.Println("Saving scrape results...") - app.Profile.SaveTweetTrove(trove) + app.Profile.SaveTweetTrove(trove, false) + go app.Profile.SaveTweetTrove(trove, true) fmt.Println("Scraping succeeded.") } @@ -107,7 +109,8 @@ func (app *Application) background_dm_polling_scrape() { trove, inbox_cursor = scraper.PollInboxUpdates(inbox_cursor) } fmt.Println("Saving DM results...") - app.Profile.SaveDMTrove(trove) + app.Profile.SaveDMTrove(trove, false) + go app.Profile.SaveDMTrove(trove, true) fmt.Println("Scraping DMs succeeded.") } diff --git a/internal/webserver/tpl/includes/author_info.tpl b/internal/webserver/tpl/includes/author_info.tpl index 479a451..bae0b98 100644 --- a/internal/webserver/tpl/includes/author_info.tpl +++ b/internal/webserver/tpl/includes/author_info.tpl @@ -3,7 +3,11 @@ diff --git a/internal/webserver/tpl/tweet_page_includes/chat_view.tpl b/internal/webserver/tpl/tweet_page_includes/chat_view.tpl index 4aa07e8..896d659 100644 --- a/internal/webserver/tpl/tweet_page_includes/chat_view.tpl +++ b/internal/webserver/tpl/tweet_page_includes/chat_view.tpl @@ -8,7 +8,11 @@
diff --git a/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl b/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl index 076f8a8..dcbd30b 100644 --- a/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl +++ b/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl @@ -57,7 +57,12 @@ {{end}} {{template "text-with-entities" $main_tweet.Text}} {{range $main_tweet.Images}} - {{end}} {{range $main_tweet.Videos}} -