From e1b29683325e722952112b1c1a0fa74ea8d26495 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 20 Aug 2023 20:45:03 -0300 Subject: [PATCH] Add subcommand to fetch home timelines (both regular and 'for you') --- cmd/twitter/main.go | 19 +++++++++++++++++-- doc/TODO.txt | 8 -------- pkg/scraper/api_graphql_utils.go | 2 ++ pkg/scraper/api_types_v2.go | 10 ++++++---- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 9f71279..2b57ebd 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -75,8 +75,9 @@ func main() { log.SetLevel(logging_level) if len(args) < 2 { - if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver") { - // "list_followed" doesn't need a target, so create a fake second arg + if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" || + args[0] == "fetch_timeline" || args[0] == "fetch_timeline_for_you") { + // Doesn't need a target, so create a fake second arg args = append(args, "") } else { die("", true, 1) @@ -138,6 +139,10 @@ func main() { fetch_user_feed(target, 999999999) case "get_user_likes": get_user_likes(target, *how_many) + case "fetch_timeline": + fetch_timeline(false) + case "fetch_timeline_for_you": + fetch_timeline(true) case "download_tweet_content": download_tweet_content(target) case "search": @@ -292,6 +297,16 @@ func get_user_likes(handle string, how_many int) { happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } +func fetch_timeline(is_for_you bool) { + trove, err := scraper.GetHomeTimeline("", is_for_you) + if err != nil { + die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2) + } + profile.SaveTweetTrove(trove) + + happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) +} + func download_tweet_content(tweet_identifier string) { tweet_id, err := extract_id_from(tweet_identifier) if err != nil { diff --git a/doc/TODO.txt b/doc/TODO.txt index 7eff01b..d0a27b4 100644 --- a/doc/TODO.txt +++ b/doc/TODO.txt @@ -3,14 +3,6 @@ User profile images and banner images => extract to table. Add `is_downloaded` TODO update-all -TODO: downloading-modes -- Could compose the following: - - Primary user's top-level tweets and threads - - Primary user's replies - - Primary user's retweets - - Parents and replies for primary user's tweets/replies - - TODO broadcast-cards panic: Unknown card type: 3691233323:periscope_broadcast panic: Unknown card type: 745291183405076480:broadcast diff --git a/pkg/scraper/api_graphql_utils.go b/pkg/scraper/api_graphql_utils.go index 66282a5..fb8e54e 100644 --- a/pkg/scraper/api_graphql_utils.go +++ b/pkg/scraper/api_graphql_utils.go @@ -58,6 +58,8 @@ type GraphqlFeatures struct { VibeApiEnabled bool `json:"vibe_api_enabled,omitempty"` InteractiveTextEnabled bool `json:"interactive_text_enabled,omitempty"` ResponsiveWebTextConversationsEnabled bool `json:"responsive_web_text_conversations_enabled"` + ResponsiveWebTwitterArticleTweetConsumptionEnabled bool `json:"responsive_web_twitter_article_tweet_consumption_enabled"` + ResponsiveWebMediaDownloadVideoEnabled bool `json:"responsive_web_media_download_video_enabled"` // Spaces Spaces2022H2Clipping bool `json:"spaces_2022_h2_clipping,omitempty"` diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index ff57fd0..6f02266 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -938,7 +938,7 @@ func GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) { return the_api.GetUserLikes(user_id, cursor) } -func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) { +func (api API) GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) { var url string body_struct := struct { Variables GraphqlVariables `json:"variables"` @@ -972,9 +972,11 @@ func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) LongformNotetweetsRichTextReadEnabled: true, LongformNotetweetsInlineMediaEnabled: true, ResponsiveWebEnhanceCardsEnabled: false, + ResponsiveWebTwitterArticleTweetConsumptionEnabled: false, + ResponsiveWebMediaDownloadVideoEnabled: false, }, } - if for_you { + if is_for_you { body_struct.QueryID = "iMKdg5Vq-ldwmiqCbvX1QA" url = "https://twitter.com/i/api/graphql/iMKdg5Vq-ldwmiqCbvX1QA/HomeLatestTimeline" } else { @@ -997,6 +999,6 @@ func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) return trove, err } -func GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) { - return the_api.GetHomeTimeline(cursor, for_you) +func GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) { + return the_api.GetHomeTimeline(cursor, is_for_you) }