Add recognizing bottom of feed

scraper/api_types_v2.go

@@ -249,6 +249,18 @@ func (api_response APIV2Response) GetCursorBottom() string {
 	return last_entry.Content.Value
 }
 
+/**
+ * Returns `true` if there's no non-cursor entries in this response, false otherwise
+ */
+func (api_response APIV2Response) IsEmpty() bool {
+	for _, e := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries {
+		if !strings.Contains(e.EntryID, "cursor") {
+			return false
+		}
+	}
+	return true
+}
+
 /**
  * Parse the collected API response and turn it into a TweetTrove
  */
@@ -343,7 +355,7 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo
 			// Empty response, cursor same as previous: end of feed has been reached
 			return END_OF_FEED
 		}
-		if len(fresh_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) == 0 {
+		if fresh_response.IsEmpty() {
 			// Response has a pinned tweet, but no other content: end of feed has been reached
 			return END_OF_FEED  // TODO: check that there actually is a pinned tweet and the request didn't just fail lol
 		}

scraper/api_types_v2_test.go

@@ -414,3 +414,31 @@ func TestParseAPIV2UserFeed(t *testing.T) {
 
 	fmt.Printf("%d Users, %d Tweets, %d Retweets\n", len(tweet_trove.Users), len(tweet_trove.Tweets), len(tweet_trove.Retweets))
 }
+
+
+/**
+ * Should correctly identify an "empty" response
+ */
+func TestAPIV2FeedIsEmpty(t *testing.T) {
+	assert := assert.New(t)
+	data, err := ioutil.ReadFile("test_responses/api_v2/empty_response.json")
+	if err != nil {
+		panic(err)
+	}
+	var feed APIV2Response
+	err = json.Unmarshal(data, &feed)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	assert.True(feed.IsEmpty())
+
+	// Make sure parsing it doesn't cause an error
+	trove, err := feed.ToTweetTrove()
+	if err != nil {
+		panic(err)
+	}
+	assert.Len(trove.Tweets, 0)
+	assert.Len(trove.Users, 0)
+	assert.Len(trove.Retweets, 0)
+}