Check for end-of-feed

scraper/api_types.go

@@ -182,6 +182,25 @@ func (t *TweetResponse) GetCursor() string {
 	return ""
 }
 
+/**
+ * Test for one case of end-of-feed.  Cursor increments on each request for some reason, but
+ * there's no new content.  This seems to happen when there's a pinned tweet.
+ *
+ * In this case, we look for an "entries" object that has only cursors in it, and no tweets.
+ */
+func (t *TweetResponse) IsEndOfFeed() bool {
+	entries := t.Timeline.Instructions[0].AddEntries.Entries
+	if len(entries) > 2 {
+		return false
+	}
+	for _, e := range entries {
+		if !strings.Contains(e.EntryID, "cursor") {
+			return false
+		}
+	}
+	return true
+}
+
 
 func idstr_to_int(idstr string) int64 {
 	id, err := strconv.Atoi(idstr)

scraper/api_types_test.go

@@ -94,3 +94,29 @@ func TestGetCursor(t *testing.T) {
 		t.Errorf("Expected %q, got %q", expected_cursor, actual_cursor)
 	}
 }
+
+
+func TestIsEndOfFeed(t *testing.T) {
+	test_cases := []struct {
+		filename string
+		is_end_of_feed bool
+	} {
+		{"test_responses/michael_malice_feed.json", false},
+		{"test_responses/kwiber_end_of_feed.json", true},
+	}
+	for _, v := range test_cases {
+		data, err := ioutil.ReadFile(v.filename)
+		if err != nil {
+			panic(err)
+		}
+		var tweet_resp scraper.TweetResponse
+		err = json.Unmarshal(data, &tweet_resp)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		result := tweet_resp.IsEndOfFeed()
+		if v.is_end_of_feed != result {
+			t.Errorf("Expected IsEndOfFeed to be %v, but got %v", v.is_end_of_feed, result)
+		}
+	}
+}