Add subcommand to fetch home timelines (both regular and 'for you')

cmd/twitter/main.go

@@ -75,8 +75,9 @@ func main() {
 	log.SetLevel(logging_level)
 
 	if len(args) < 2 {
-		if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver") {
+		if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" ||
-			// "list_followed" doesn't need a target, so create a fake second arg
+			args[0] == "fetch_timeline" || args[0] == "fetch_timeline_for_you") {
+			// Doesn't need a target, so create a fake second arg
 			args = append(args, "")
 		} else {
 			die("", true, 1)
@@ -138,6 +139,10 @@ func main() {
 		fetch_user_feed(target, 999999999)
 	case "get_user_likes":
 		get_user_likes(target, *how_many)
+	case "fetch_timeline":
+		fetch_timeline(false)
+	case "fetch_timeline_for_you":
+		fetch_timeline(true)
 	case "download_tweet_content":
 		download_tweet_content(target)
 	case "search":
@@ -292,6 +297,16 @@ func get_user_likes(handle string, how_many int) {
 	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
 }
 
+func fetch_timeline(is_for_you bool) {
+	trove, err := scraper.GetHomeTimeline("", is_for_you)
+	if err != nil {
+		die(fmt.Sprintf("Error fetching timeline:\n  %s", err.Error()), false, -2)
+	}
+	profile.SaveTweetTrove(trove)
+
+	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
+}
+
 func download_tweet_content(tweet_identifier string) {
 	tweet_id, err := extract_id_from(tweet_identifier)
 	if err != nil {

doc/TODO.txt

@@ -3,14 +3,6 @@ User profile images and banner images => extract to table.  Add `is_downloaded`
 
 TODO update-all
 
-TODO: downloading-modes
-- Could compose the following:
-	- Primary user's top-level tweets and threads
-	- Primary user's replies
-	- Primary user's retweets
-	- Parents and replies for primary user's tweets/replies
-
-
 TODO broadcast-cards
 panic: Unknown card type: 3691233323:periscope_broadcast
 panic: Unknown card type: 745291183405076480:broadcast

pkg/scraper/api_graphql_utils.go

@@ -58,6 +58,8 @@ type GraphqlFeatures struct {
 	VibeApiEnabled                                                 bool `json:"vibe_api_enabled,omitempty"`
 	InteractiveTextEnabled                                         bool `json:"interactive_text_enabled,omitempty"`
 	ResponsiveWebTextConversationsEnabled                          bool `json:"responsive_web_text_conversations_enabled"`
+	ResponsiveWebTwitterArticleTweetConsumptionEnabled             bool `json:"responsive_web_twitter_article_tweet_consumption_enabled"`
+	ResponsiveWebMediaDownloadVideoEnabled                         bool `json:"responsive_web_media_download_video_enabled"`
 
 	// Spaces
 	Spaces2022H2Clipping          bool `json:"spaces_2022_h2_clipping,omitempty"`

pkg/scraper/api_types_v2.go

@@ -938,7 +938,7 @@ func GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) {
 	return the_api.GetUserLikes(user_id, cursor)
 }
 
-func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) {
+func (api API) GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) {
 	var url string
 	body_struct := struct {
 		Variables GraphqlVariables `json:"variables"`
@@ -972,9 +972,11 @@ func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error)
 			LongformNotetweetsRichTextReadEnabled:                          true,
 			LongformNotetweetsInlineMediaEnabled:                           true,
 			ResponsiveWebEnhanceCardsEnabled:                               false,
+			ResponsiveWebTwitterArticleTweetConsumptionEnabled:             false,
+			ResponsiveWebMediaDownloadVideoEnabled:                         false,
 		},
 	}
-	if for_you {
+	if is_for_you {
 		body_struct.QueryID = "iMKdg5Vq-ldwmiqCbvX1QA"
 		url = "https://twitter.com/i/api/graphql/iMKdg5Vq-ldwmiqCbvX1QA/HomeLatestTimeline"
 	} else {
@@ -997,6 +999,6 @@ func (api API) GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error)
 	return trove, err
 }
 
-func GetHomeTimeline(cursor string, for_you bool) (TweetTrove, error) {
+func GetHomeTimeline(cursor string, is_for_you bool) (TweetTrove, error) {
-	return the_api.GetHomeTimeline(cursor, for_you)
+	return the_api.GetHomeTimeline(cursor, is_for_you)
 }