Avoid crashing on tombstones in tweet detail replies

scraper/api_types_v2.go

@@ -438,7 +438,10 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
 					continue
 				}
 				trove, err := item.Item.ItemContent.TweetResults.ToTweetTrove()
-				if err != nil {
+				if errors.Is(err, ErrorIsTombstone) {
+					// TODO: do something with tombstones in replies to a Tweet Detail
+					// For now, just ignore tombstones in the replies
+				} else if err != nil {
 					panic(err)
 				}
 				ret.MergeWith(trove)

scraper/api_types_v2_test.go

@@ -798,3 +798,20 @@ func TestTweetDetailWithShowMoreButton(t *testing.T) {
 	// Test the "Show more replies" cursor
 	assert.Equal("NwAAAPANHBlWhMC--YvZ3a8ugsDS1aGWqbAugIDQ6dCa3xIAYL314NGqsAkA0OGFzYqwLiUEERUOAAA", resp.GetCursorBottom())
 }
+
+func TestConversationThreadWithTombstoneReplies(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+	data, err := os.ReadFile("test_responses/api_v2/tweet_detail_with_tombstone_replies.json")
+	require.NoError(err)
+	var resp APIV2Response
+	err = json.Unmarshal(data, &resp)
+	require.NoError(err)
+
+	trove, err := resp.ToTweetTrove()
+	require.NoError(err)
+
+	t1, is_ok := trove.Tweets[1433713164546293767] // Main tweet
+	assert.True(is_ok)
+	assert.False(t1.IsStub)
+}

scraper/test_responses/api_v2/tweet_detail_with_tombstone_replies.json

@@ -0,0 +1 @@
+{"data":{"threaded_conversation_with_injections_v2":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1433713164546293767","sortIndex":"7789658872308482040","content":{"entryType":"TimelineTimelineItem","__typename":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1433713164546293767","has_birdwatch_notes":true,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDE5NjM5Nw==","rest_id":"44196397","affiliates_highlighted_label":{"label":{"url":{"url":"https://twitter.com/Twitter","urlType":"DeepLink"},"badge":{"url":"https://pbs.twimg.com/profile_images/1488548719062654976/u6qfBBkF_bigger.jpg"},"description":"Twitter","userLabelType":"BusinessLabel","userLabelDisplayType":"Badge"}},"is_blue_verified":true,"profile_image_shape":"Circle","legacy":{"created_at":"Tue Jun 02 20:12:29 +0000 2009","default_profile":false,"default_profile_image":false,"description":"","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":25959,"followers_count":143969675,"friends_count":333,"has_custom_timelines":true,"is_translator":false,"listed_count":123726,"location":"","media_count":1591,"name":"Elon Musk","normal_followers_count":143969675,"pinned_tweet_ids_str":[],"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/44196397/1576183471","profile_image_url_https":"https://pbs.twimg.com/profile_images/1590968738358079488/IY9Gx6Ok_normal.jpg","profile_interstitial_type":"","screen_name":"elonmusk","statuses_count":27154,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1433713164546293767"],"editable_until_msecs":"1630660630187","is_edit_eligible":true,"edits_remaining":"5"},"is_translatable":false,"views":{"state":"Enabled"},"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","legacy":{"bookmark_count":2244,"bookmarked":false,"created_at":"Fri Sep 03 08:47:10 +0000 2021","conversation_id_str":"1433713164546293767","display_text_range":[0,29],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":447191,"favorited":false,"full_text":"Time is the ultimate currency","is_quote_status":false,"lang":"en","quote_count":6922,"reply_count":30446,"retweet_count":65262,"retweeted":false,"user_id_str":"44196397","id_str":"1433713164546293767"},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"Tweet","hasModeratedReplies":false}}},{"entryId":"conversationthread-1433713505291497472","sortIndex":"7789658872308482030","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1433713505291497472-tweet-1433713505291497472","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"TweetTombstone","tombstone":{"__typename":"TextTombstone","text":{"rtl":false,"text":"This Tweet is unavailable. Learn more","entities":[{"fromIndex":27,"toIndex":37,"ref":{"type":"TimelineUrl","url":"https://help.twitter.com/rules-and-policies/notices-on-twitter","urlType":"ExternalUrl"}}]}}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"conversationthread-1433713505291497472-tweet-1433713527542280194","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"TweetTombstone","tombstone":{"__typename":"TextTombstone","text":{"rtl":false,"text":"This Tweet is unavailable. Learn more","entities":[{"fromIndex":27,"toIndex":37,"ref":{"type":"TimelineUrl","url":"https://help.twitter.com/rules-and-policies/notices-on-twitter","urlType":"ExternalUrl"}}]}}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"conversationthread-1433713505291497472-cursor-showmore-7468565295226794256","item":{"itemContent":{"itemType":"TimelineTimelineCursor","__typename":"TimelineTimelineCursor","value":"PAAAAPAtPBwcFoTAo-3ttsnlJxUCAAAYJmNvbnZlcnNhdGlvbnRocmVhZC0xNDMzNzEzNTA1MjkxNDk3NDcyIgAA","cursorType":"ShowMore","displayTreatment":{"actionText":"Show replies"}},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}}]},{"type":"TimelineTerminateTimeline","direction":"Top"}]}}}