From 018df0d6a44556210e71012c7aeeed50b5b49068 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 18 Feb 2024 14:47:12 -0800 Subject: [PATCH] Rename 'for you' timeline to 'following only' to match Twitter's upstream changes - Also add 'get_user_likes_all' cmd subcommand to get all users likes --- cmd/twitter/help_message.txt | 15 +++++++++++++++ cmd/twitter/main.go | 12 +++++++----- internal/webserver/stopwatch.go | 6 +++--- pkg/scraper/api_types_v2.go | 8 ++++---- pkg/scraper/api_types_v2_test.go | 4 ++-- ...you.json => home_timeline_following_only.json} | 0 6 files changed, 31 insertions(+), 14 deletions(-) rename pkg/scraper/test_responses/api_v2/{home_timeline_for_you.json => home_timeline_following_only.json} (100%) diff --git a/cmd/twitter/help_message.txt b/cmd/twitter/help_message.txt index 93c76cb..6dc92c0 100644 --- a/cmd/twitter/help_message.txt +++ b/cmd/twitter/help_message.txt @@ -50,6 +50,21 @@ This application downloads tweets from twitter and saves them in a SQLite databa Gets the most recent ~50 tweets. If "get_user_tweets_all" is used, gets up to ~3200 tweets (API limit). + get_user_likes + get_user_likes_all + is the user handle. + Gets the most recent ~50 "likes". + If "get_user_tweets_all" is used, gets up to ~3200 "liked" tweets (API limit). + + fetch_timeline + fetch_timeline_following_only + Fetch the logged-in user's timeline (home feed). + No is needed; will be ignored if given. + Twitter keeps renaming the feeds, but the two options are: + - "fetch_timeline" uses Twitter's "algorithm", including recommendations, promoted content, etc + - "fetch_timeline_following_only" is just tweets from people you follow, in the order they were + posted (skips the "algorithm") + follow unfollow is the user handle diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 616058f..783d48f 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -72,7 +72,7 @@ func main() { if len(args) < 2 { if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" || args[0] == "fetch_timeline" || - args[0] == "fetch_timeline_for_you" || args[0] == "fetch_inbox") { + args[0] == "fetch_timeline_following_only" || args[0] == "fetch_inbox") { // Doesn't need a target, so create a fake second arg args = append(args, "") } else { @@ -135,13 +135,15 @@ func main() { fetch_user_feed(target, 999999999) case "get_user_likes": get_user_likes(target, *how_many) + case "get_user_likes_all": + get_user_likes(target, 999999999) case "get_followers": get_followers(target, *how_many) case "get_followees": get_followees(target, *how_many) case "fetch_timeline": - fetch_timeline(false) - case "fetch_timeline_for_you": + fetch_timeline(false) // TODO: *how_many + case "fetch_timeline_following_only": fetch_timeline(true) case "download_tweet_content": download_tweet_content(target) @@ -344,8 +346,8 @@ func get_followers(handle string, how_many int) { happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users))) } -func fetch_timeline(is_for_you bool) { - trove, err := scraper.GetHomeTimeline("", is_for_you) +func fetch_timeline(is_following_only bool) { + trove, err := scraper.GetHomeTimeline("", is_following_only) if err != nil { die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2) } diff --git a/internal/webserver/stopwatch.go b/internal/webserver/stopwatch.go index 499b2fb..dea68fb 100644 --- a/internal/webserver/stopwatch.go +++ b/internal/webserver/stopwatch.go @@ -6,7 +6,7 @@ import ( "time" ) -var is_for_you_only = true // Do one initial scrape of the "for you" feed and then just regular feed after that +var is_following_only = true // Do one initial scrape of the "following_only" feed and then just regular feed after that func (app *Application) background_scrape() { // Avoid crashing the thread if a scrape fails @@ -31,7 +31,7 @@ func (app *Application) background_scrape() { } fmt.Println("Scraping home timeline...") - trove, err := scraper.GetHomeTimeline("", is_for_you_only) + trove, err := scraper.GetHomeTimeline("", is_following_only) if err != nil { app.ErrorLog.Printf("Background scrape failed: %s", err.Error()) return @@ -40,7 +40,7 @@ func (app *Application) background_scrape() { app.Profile.SaveTweetTrove(trove, false) go app.Profile.SaveTweetTrove(trove, true) fmt.Println("Scraping succeeded.") - is_for_you_only = false + is_following_only = false } func (app *Application) background_user_likes_scrape() { diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index bd02493..b0760b1 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -1053,7 +1053,7 @@ func GetUserLikes(user_id UserID, how_many int) (TweetTrove, error) { return the_api.GetPaginatedQuery(PaginatedUserLikes{user_id}, how_many) } -func (api *API) GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) { +func (api *API) GetHomeTimeline(cursor string, is_following_only bool) (TweetTrove, error) { var url string body_struct := struct { Variables GraphqlVariables `json:"variables"` @@ -1091,7 +1091,7 @@ func (api *API) GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, err ResponsiveWebMediaDownloadVideoEnabled: false, }, } - if is_for_you { + if is_following_only { body_struct.QueryID = "iMKdg5Vq-ldwmiqCbvX1QA" url = "https://twitter.com/i/api/graphql/iMKdg5Vq-ldwmiqCbvX1QA/HomeLatestTimeline" } else { @@ -1114,8 +1114,8 @@ func (api *API) GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, err return trove, err } -func GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) { - return the_api.GetHomeTimeline(cursor, is_for_you) +func GetHomeTimeline(cursor string, is_following_only bool) (TweetTrove, error) { + return the_api.GetHomeTimeline(cursor, is_following_only) } func (api API) GetUser(handle UserHandle) (APIUser, error) { diff --git a/pkg/scraper/api_types_v2_test.go b/pkg/scraper/api_types_v2_test.go index 8509f14..8bc64e2 100644 --- a/pkg/scraper/api_types_v2_test.go +++ b/pkg/scraper/api_types_v2_test.go @@ -903,9 +903,9 @@ func TestParseHomeTimeline(t *testing.T) { require.Len(trove.Users, 11) } -func TestParseHomeTimelineForYou(t *testing.T) { +func TestParseHomeTimelineFollowingOnly(t *testing.T) { require := require.New(t) - data, err := os.ReadFile("test_responses/api_v2/home_timeline_for_you.json") + data, err := os.ReadFile("test_responses/api_v2/home_timeline_following_only.json") require.NoError(err) var response_result APIV2Response err = json.Unmarshal(data, &response_result) diff --git a/pkg/scraper/test_responses/api_v2/home_timeline_for_you.json b/pkg/scraper/test_responses/api_v2/home_timeline_following_only.json similarity index 100% rename from pkg/scraper/test_responses/api_v2/home_timeline_for_you.json rename to pkg/scraper/test_responses/api_v2/home_timeline_following_only.json