Avoid crashing on inline labels like 'More Replies'

scraper/api_types_v2.go

@@ -421,6 +421,9 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
 		// - e.Content.EntryType -> User Feed itself
 		// - e.Content.ItemContent.ItemType -> conversation thread in a user feed
 		return NewTweetTrove()
+	} else if e.Content.ItemContent.ItemType == "TimelineLabel" {
+		// Skip inline "labels" like "More Replies" that appear when you click "show more replies"
+		return NewTweetTrove()
 	} else if e.Content.EntryType == "TimelineTimelineModule" {
 		ret := NewTweetTrove()
 

scraper/api_types_v2_test.go

@@ -765,3 +765,23 @@ func TestConversationThreadEntryWithShowMoreButton(t *testing.T) {
 
 	assert.Len(trove.Users, 1)
 }
+
+func TestTweetWithInlineLabel(t *testing.T) {
+	assert := assert.New(t)
+	data, err := os.ReadFile("test_responses/api_v2/response_with_inline_label.json")
+	require.NoError(t, err)
+	var resp APIV2Response
+
+	err = json.Unmarshal(data, &resp)
+	require.NoError(t, err)
+
+	// Test the "Show additional replies, including offensive" cursor
+	assert.Equal("NwAAAPAoHBlWiICz7Y27o4kploCzmY7CyYgphoC-7d65sYgphsCj7cml4ogpjMC4ue3954gpJQYRFRAAAA", resp.GetCursorBottom())
+
+	trove, err := resp.ToTweetTrove()
+	assert.NoError(err)
+
+	assert.Len(trove.Retweets, 0)
+	assert.Len(trove.Tweets, 1)
+	assert.Len(trove.Users, 1)
+}

scraper/test_responses/api_v2/response_with_inline_label.json

@@ -0,0 +1 @@
+{"data":{"threaded_conversation_with_injections_v2":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"label-7743831484773449677","sortIndex":"7743831484773449677","content":{"entryType":"TimelineTimelineItem","__typename":"TimelineTimelineItem","itemContent":{"itemType":"TimelineLabel","__typename":"TimelineLabel","text":"More Replies","display_type":"InlineHeader"}}},{"entryId":"conversationthread-1479594142065909771","sortIndex":"7743831484773449672","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1479594142065909771-tweet-1479594142065909771","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1479594142065909771","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDAwMzk0NTMzMDI4OTU4MjA5","rest_id":"1400394533028958209","affiliates_highlighted_label":{},"is_blue_verified":false,"profile_image_shape":"Circle","legacy":{"created_at":"Thu Jun 03 10:11:00 +0000 2021","default_profile":true,"default_profile_image":false,"description":"I’ve been watching for too long. time to do something worth watching. I hoist the black flag with saliva on my hands","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":1437,"followers_count":27,"friends_count":51,"has_custom_timelines":true,"is_translator":false,"listed_count":0,"location":"Mindfulness ","media_count":10,"name":"SimplyNotSimple","normal_followers_count":27,"pinned_tweet_ids_str":["1543627879770120199"],"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1400394533028958209/1626436891","profile_image_url_https":"https://pbs.twimg.com/profile_images/1415997462452609025/7T8Lf5Fv_normal.jpg","profile_interstitial_type":"","screen_name":"NotSimplySimple","statuses_count":1710,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1479594142065909771"],"editable_until_msecs":"1641599507655","is_edit_eligible":true,"edits_remaining":"5"},"is_translatable":false,"views":{"state":"Enabled"},"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","legacy":{"bookmark_count":0,"bookmarked":false,"created_at":"Fri Jan 07 23:21:47 +0000 2022","conversation_id_str":"1479540552081326085","display_text_range":[15,118],"entities":{"user_mentions":[{"id_str":"44067298","name":"Michael Malice","screen_name":"michaelmalice","indices":[0,14]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":0,"favorited":false,"full_text":"@michaelmalice I love the idea of punishment they think they create by blocking people who do not care what they think","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1479540552081326085","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"1400394533028958209","id_str":"1479594142065909771"},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"LowQuality"},"timelinesDetails":{"controllerData":"DAACDAAEDAABCgABFSACCCAHAAUKAAIAAAAACADACAAAAAA="}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"LowQuality"}}}}},{"entryId":"cursor-showmorethreadsprompt-3117044691777571989","sortIndex":"7743831484773449671","content":{"entryType":"TimelineTimelineItem","__typename":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTimelineCursor","__typename":"TimelineTimelineCursor","value":"NwAAAPAoHBlWiICz7Y27o4kploCzmY7CyYgphoC-7d65sYgphsCj7cml4ogpjMC4ue3954gpJQYRFRAAAA","cursorType":"ShowMoreThreadsPrompt","displayTreatment":{"actionText":"Show","labelText":"Show additional replies, including those that may contain offensive content"}}}}]},{"type":"TimelineTerminateTimeline","direction":"Top"}]}}}