diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 509b60e..5bbf7b8 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -249,6 +249,18 @@ func (api_response APIV2Response) GetCursorBottom() string { return last_entry.Content.Value } +/** + * Returns `true` if there's no non-cursor entries in this response, false otherwise + */ +func (api_response APIV2Response) IsEmpty() bool { + for _, e := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries { + if !strings.Contains(e.EntryID, "cursor") { + return false + } + } + return true +} + /** * Parse the collected API response and turn it into a TweetTrove */ @@ -343,7 +355,7 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo // Empty response, cursor same as previous: end of feed has been reached return END_OF_FEED } - if len(fresh_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) == 0 { + if fresh_response.IsEmpty() { // Response has a pinned tweet, but no other content: end of feed has been reached return END_OF_FEED // TODO: check that there actually is a pinned tweet and the request didn't just fail lol } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index 557dd60..a4d0634 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -414,3 +414,31 @@ func TestParseAPIV2UserFeed(t *testing.T) { fmt.Printf("%d Users, %d Tweets, %d Retweets\n", len(tweet_trove.Users), len(tweet_trove.Tweets), len(tweet_trove.Retweets)) } + + +/** + * Should correctly identify an "empty" response + */ +func TestAPIV2FeedIsEmpty(t *testing.T) { + assert := assert.New(t) + data, err := ioutil.ReadFile("test_responses/api_v2/empty_response.json") + if err != nil { + panic(err) + } + var feed APIV2Response + err = json.Unmarshal(data, &feed) + if err != nil { + t.Errorf(err.Error()) + } + + assert.True(feed.IsEmpty()) + + // Make sure parsing it doesn't cause an error + trove, err := feed.ToTweetTrove() + if err != nil { + panic(err) + } + assert.Len(trove.Tweets, 0) + assert.Len(trove.Users, 0) + assert.Len(trove.Retweets, 0) +}