Add parsing of HomeTimeline (not For You)

pkg/scraper/api_types_v2.go

@@ -428,10 +428,12 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
 		ret := NewTweetTrove()
 
 		parts := strings.Split(e.EntryID, "-")
-		if parts[0] == "homeConversation" || parts[0] == "conversationthread" || strings.Join(parts[0:2], "-") == "profile-conversation" {
+		if parts[0] == "homeConversation" || parts[0] == "conversationthread" ||
+			strings.Join(parts[0:2], "-") == "profile-conversation" || strings.Join(parts[0:2], "-") == "home-conversation" {
 			// Process it.
 			// - "profile-conversation": conversation thread on a user feed
 			// - "homeConversation": This looks like it got changed to "profile-conversation"
+			// - "home-conversation": probably same as above lol-- someone did some refactoring
 			// - "conversationthread": conversation thread in the replies under a TweetDetail view
 			for _, item := range e.Content.Items {
 				if item.Item.ItemContent.ItemType == "TimelineTimelineCursor" {
@@ -504,6 +506,11 @@ type APIV2Instruction struct {
 
 type APIV2Response struct {
 	Data struct {
+		Home struct {
+			HomeTimelineUrt struct {
+				Instructions []APIV2Instruction `json:"instructions"`
+			} `json:"home_timeline_urt"`
+		} `json:"home"`
 		User struct {
 			Result struct {
 				TimelineV2 struct { // "Likes" feed calls this "timeline_v2" for some reason
@@ -543,6 +550,12 @@ func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction {
 			return &instructions[i]
 		}
 	}
+	instructions = api_response.Data.Home.HomeTimelineUrt.Instructions
+	for i := range instructions {
+		if instructions[i].Type == "TimelineAddEntries" {
+			return &instructions[i]
+		}
+	}
 	panic("No 'TimelineAddEntries' found")
 }
 
@@ -924,3 +937,60 @@ func (api API) GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) {
 func GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) {
 	return the_api.GetUserLikes(user_id, cursor)
 }
+
+func (api API) GetHomeTimeline(cursor string) (TweetTrove, error) {
+	url := "https://twitter.com/i/api/graphql/W4Tpu1uueTGK53paUgxF0Q/HomeTimeline"
+	body_struct := struct {
+		Variables GraphqlVariables `json:"variables"`
+		Features  GraphqlFeatures  `json:"features"`
+		QueryID   string           `json:"queryId"`
+	}{
+		Variables: GraphqlVariables{
+			Count:                  40,
+			Cursor:                 cursor,
+			IncludePromotedContent: false,
+			// LatestControlAvailable: true, // TODO: new field?
+			WithCommunity: true,
+			// SeenTweetIDs: []string{"...some TweetIDs"}? // TODO: new field?
+		},
+		Features: GraphqlFeatures{
+			RWebListsTimelineRedesignEnabled:                               true,
+			ResponsiveWebGraphqlExcludeDirectiveEnabled:                    true,
+			VerifiedPhoneLabelEnabled:                                      false,
+			CreatorSubscriptionsTweetPreviewApiEnabled:                     true,
+			ResponsiveWebGraphqlTimelineNavigationEnabled:                  true,
+			ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled:      false,
+			TweetypieUnmentionOptimizationEnabled:                          true,
+			ResponsiveWebEditTweetApiEnabled:                               true,
+			GraphqlIsTranslatableRWebTweetIsTranslatableEnabled:            true,
+			ViewCountsEverywhereApiEnabled:                                 true,
+			LongformNotetweetsConsumptionEnabled:                           true,
+			TweetAwardsWebTippingEnabled:                                   false,
+			FreedomOfSpeechNotReachFetchEnabled:                            false,
+			StandardizedNudgesMisinfo:                                      true,
+			TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true,
+			LongformNotetweetsRichTextReadEnabled:                          true,
+			LongformNotetweetsInlineMediaEnabled:                           true,
+			ResponsiveWebEnhanceCardsEnabled:                               false,
+		},
+		QueryID: "W4Tpu1uueTGK53paUgxF0Q",
+	}
+	var response APIV2Response
+	body_bytes, err := json.Marshal(body_struct)
+	if err != nil {
+		panic(err)
+	}
+	err = api.do_http_POST(url, string(body_bytes), &response)
+	if err != nil {
+		panic(err)
+	}
+	trove, err := response.ToTweetTrove()
+	if err != nil {
+		return TweetTrove{}, err
+	}
+	return trove, err
+}
+
+func GetHomeTimeline(cursor string) (TweetTrove, error) {
+	return the_api.GetHomeTimeline(cursor)
+}

pkg/scraper/api_types_v2_test.go

@@ -1,9 +1,10 @@
 package scraper_test
 
 import (
+	"testing"
+
 	"encoding/json"
 	"os"
-	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -867,6 +868,21 @@ func TestTweetDetailWithUnjoinedNontombstoneTweet(t *testing.T) {
 	assert.Equal(t2.ID, t3.InReplyToID)
 }
 
+func TestParseHomeTimeline(t *testing.T) {
+	require := require.New(t)
+	data, err := os.ReadFile("test_responses/api_v2/home_timeline.json")
+	require.NoError(err)
+	var response_result APIV2Response
+	err = json.Unmarshal(data, &response_result)
+	require.NoError(err)
+
+	trove, err := response_result.ToTweetTrove()
+	require.NoError(err)
+
+	require.Len(trove.Tweets, 13)
+	require.Len(trove.Users, 11)
+}
+
 func TestParseResultAsLikes(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)