diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 14c476c..83266fd 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -276,16 +276,18 @@ type APIV2Entry struct { } `json:"content"` } +type APIV2Instruction struct { + Type string `json:"type"` + Entries []APIV2Entry`json:"entries"` +} + type APIV2Response struct { Data struct { User struct { Result struct { Timeline struct { Timeline struct { - Instructions []struct { - Type string `json:"type"` - Entries []APIV2Entry`json:"entries"` - } `json:"instructions"` + Instructions []APIV2Instruction `json:"instructions"` } `json:"timeline"` } `json:"timeline"` } `json:"result"` @@ -293,8 +295,18 @@ type APIV2Response struct { } `json:"data"` } +func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction { + instructions := api_response.Data.User.Result.Timeline.Timeline.Instructions + for i := range instructions { + if instructions[i].Type == "TimelineAddEntries" { + return &instructions[i] + } + } + panic("No 'TimelineAddEntries' found") +} + func (api_response APIV2Response) GetCursorBottom() string { - entries := api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries + entries := api_response.GetMainInstruction().Entries last_entry := entries[len(entries) - 1] if last_entry.Content.CursorType != "Bottom" { panic("No bottom cursor found") @@ -307,7 +319,7 @@ func (api_response APIV2Response) GetCursorBottom() string { * Returns `true` if there's no non-cursor entries in this response, false otherwise */ func (api_response APIV2Response) IsEmpty() bool { - for _, e := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries { + for _, e := range api_response.GetMainInstruction().Entries { if !strings.Contains(e.EntryID, "cursor") { return false } @@ -320,7 +332,7 @@ func (api_response APIV2Response) IsEmpty() bool { */ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { ret := NewTweetTrove() - for _, entry := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries { // TODO: the second Instruction is the pinned tweet + for _, entry := range api_response.GetMainInstruction().Entries { // TODO: the second Instruction is the pinned tweet if !strings.HasPrefix(entry.EntryID, "tweet-") { continue } @@ -399,13 +411,13 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo // TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop // forever getting no new tweets last_response := response - for last_response.GetCursorBottom() != "" && len(response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) < min_tweets { + for last_response.GetCursorBottom() != "" && len(response.GetMainInstruction().Entries) < min_tweets { fresh_response, err := api.GetGraphqlFeedFor(user_id, last_response.GetCursorBottom()) if err != nil { return err } - if fresh_response.GetCursorBottom() == last_response.GetCursorBottom() && len(fresh_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) == 0 { + if fresh_response.GetCursorBottom() == last_response.GetCursorBottom() && len(fresh_response.GetMainInstruction().Entries) == 0 { // Empty response, cursor same as previous: end of feed has been reached return END_OF_FEED } @@ -417,11 +429,11 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo last_response = &fresh_response // Copy over the entries - response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries = append( - response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries, - last_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries...) + response.GetMainInstruction().Entries = append( + response.GetMainInstruction().Entries, + last_response.GetMainInstruction().Entries...) - fmt.Printf("Have %d entries so far\n", len(response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries)) + fmt.Printf("Have %d entries so far\n", len(response.GetMainInstruction().Entries)) } return nil } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index 7ad1517..5ad3ebe 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -516,3 +516,26 @@ func TestAPIV2FeedIsEmpty(t *testing.T) { assert.Len(trove.Users, 0) assert.Len(trove.Retweets, 0) } + +/** + * Should get the right Instruction element + */ +func TestAPIV2GetMainInstructionFromFeed(t *testing.T) { + assert := assert.New(t) + data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json") + if err != nil { + panic(err) + } + var feed APIV2Response + err = json.Unmarshal(data, &feed) + if err != nil { + t.Errorf(err.Error()) + } + + assert.Equal(len(feed.GetMainInstruction().Entries), 41) + + // Test that this is a writable version + feed.GetMainInstruction().Entries = append(feed.GetMainInstruction().Entries, APIV2Entry{EntryID: "asdf"}) + assert.Equal(len(feed.GetMainInstruction().Entries), 42) + assert.Equal(feed.GetMainInstruction().Entries[41].EntryID, "asdf") +}