diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index fa23981..44ee8b7 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -541,7 +541,12 @@ func (api_response APIV2Response) GetCursorBottom() string { } // For a Tweet Detail page: - if entry.Content.ItemContent.CursorType == "Bottom" { + if entry.Content.ItemContent.CursorType == "Bottom" || + entry.Content.ItemContent.CursorType == "ShowMoreThreadsPrompt" || + entry.Content.ItemContent.CursorType == "ShowMoreThreads" { + // "Bottom": normal cursor, auto-loads when it scrolls into view + // "ShowMoreThreads": normal cursor, but you have to click it to load more + // "ShowMoreThreadsPrompt": show offensive/low quality replies return entry.Content.ItemContent.Value } } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index fbf865c..8916e9e 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -785,3 +785,16 @@ func TestTweetWithInlineLabel(t *testing.T) { assert.Len(trove.Tweets, 1) assert.Len(trove.Users, 1) } + +func TestTweetDetailWithShowMoreButton(t *testing.T) { + assert := assert.New(t) + data, err := os.ReadFile("test_responses/api_v2/tweet_detail_with_show_more_replies_button.json") + require.NoError(t, err) + var resp APIV2Response + + err = json.Unmarshal(data, &resp) + require.NoError(t, err) + + // Test the "Show more replies" cursor + assert.Equal("NwAAAPANHBlWhMC--YvZ3a8ugsDS1aGWqbAugIDQ6dCa3xIAYL314NGqsAkA0OGFzYqwLiUEERUOAAA", resp.GetCursorBottom()) +} diff --git a/scraper/test_responses/api_v2/tweet_detail_with_show_more_replies_button.json b/scraper/test_responses/api_v2/tweet_detail_with_show_more_replies_button.json new file mode 100644 index 0000000..1826259 --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_detail_with_show_more_replies_button.json @@ -0,0 +1 @@ +{"data":{"threaded_conversation_with_injections_v2":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1670759701070458880","sortIndex":"7552612335784316927","content":{"entryType":"TimelineTimelineItem","__typename":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1670759701070458880","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDE1MzE3MjI0NTI5NzkzMDI4","rest_id":"1415317224529793028","affiliates_highlighted_label":{},"is_blue_verified":false,"profile_image_shape":"Circle","legacy":{"created_at":"Wed Jul 14 14:28:51 +0000 2021","default_profile":true,"default_profile_image":false,"description":"\"What would I have done? I would have a done a legitimate investigation to find out what exactly happened in 1099.\"\n\ncheck out eggreport on sûbstäck if you want","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":49651,"followers_count":15565,"friends_count":681,"has_custom_timelines":true,"is_translator":false,"listed_count":32,"location":"europooristan ","media_count":3611,"name":"garfieldbot","normal_followers_count":15565,"pinned_tweet_ids_str":["1554817743588835333"],"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1415317224529793028/1642757653","profile_image_url_https":"https://pbs.twimg.com/profile_images/1653404132017938433/KdcqbR_6_normal.jpg","profile_interstitial_type":"","screen_name":"cumlasagna1","statuses_count":50570,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1670759701070458880"],"editable_until_msecs":"1687178728000","is_edit_eligible":false,"edits_remaining":"5"},"is_translatable":false,"views":{"count":"6985","state":"EnabledWithCount"},"source":"Twitter Web App","legacy":{"bookmark_count":2,"bookmarked":false,"created_at":"Mon Jun 19 11:45:28 +0000 2023","conversation_id_str":"1670759701070458880","display_text_range":[0,204],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":95,"favorited":false,"full_text":"i endorse rfk and its not even because of the wax stuff, its because i think he's the best chance you have at a friendly member of the nobility caste, who would potentially have a sense of noblisse oblige","is_quote_status":false,"lang":"en","quote_count":2,"reply_count":6,"retweet_count":1,"retweeted":false,"user_id_str":"1415317224529793028","id_str":"1670759701070458880","self_thread":{"id_str":"1670759701070458880"}},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"SelfThread","hasModeratedReplies":false}}},{"entryId":"conversationthread-1670760026552651778","sortIndex":"7552612335784316917","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1670760026552651778-tweet-1670760026552651778","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1670760026552651778","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDE1MzE3MjI0NTI5NzkzMDI4","rest_id":"1415317224529793028","affiliates_highlighted_label":{},"is_blue_verified":false,"profile_image_shape":"Circle","legacy":{"created_at":"Wed Jul 14 14:28:51 +0000 2021","default_profile":true,"default_profile_image":false,"description":"\"What would I have done? I would have a done a legitimate investigation to find out what exactly happened in 1099.\"\n\ncheck out eggreport on sûbstäck if you want","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":49651,"followers_count":15565,"friends_count":681,"has_custom_timelines":true,"is_translator":false,"listed_count":32,"location":"europooristan ","media_count":3611,"name":"garfieldbot","normal_followers_count":15565,"pinned_tweet_ids_str":["1554817743588835333"],"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1415317224529793028/1642757653","profile_image_url_https":"https://pbs.twimg.com/profile_images/1653404132017938433/KdcqbR_6_normal.jpg","profile_interstitial_type":"","screen_name":"cumlasagna1","statuses_count":50570,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1670760026552651778"],"editable_until_msecs":"1687178806000","is_edit_eligible":false,"edits_remaining":"5"},"is_translatable":false,"views":{"count":"1108","state":"EnabledWithCount"},"source":"Twitter Web App","legacy":{"bookmark_count":0,"bookmarked":false,"created_at":"Mon Jun 19 11:46:46 +0000 2023","conversation_id_str":"1670759701070458880","display_text_range":[0,229],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":31,"favorited":false,"full_text":"although nothing is getting solved until younger people take control of the world, ideally you would want a 40 year old with rfks profile. perhaps even yunger. but if you have to have an old boomer then hes probably your best bet","in_reply_to_screen_name":"cumlasagna1","in_reply_to_status_id_str":"1670759701070458880","in_reply_to_user_id_str":"1415317224529793028","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":1,"retweet_count":0,"retweeted":false,"user_id_str":"1415317224529793028","id_str":"1670760026552651778","self_thread":{"id_str":"1670759701070458880"}},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"SelfThread"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"},"timelinesDetails":{"controllerData":"DAACDAAEDAABCgABFQgCCCADg4UKAAIAAAAACADACAAAAAA="}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"conversationthread-1670926004184043521","sortIndex":"7552612335784316907","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1670926004184043521-tweet-1670926004184043521","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1670926004184043521","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDk5OTUwMTgyODQyMjE2NDU4","rest_id":"1499950182842216458","affiliates_highlighted_label":{},"is_blue_verified":true,"profile_image_shape":"Circle","legacy":{"created_at":"Sat Mar 05 03:29:59 +0000 2022","default_profile":true,"default_profile_image":false,"description":"Abercrombie and Fitch Nationalist","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":66353,"followers_count":566,"friends_count":675,"has_custom_timelines":true,"is_translator":false,"listed_count":6,"location":"","media_count":348,"name":"Dr Ted, CPA, MBA, ASPCA, LGBTQ, Esq","normal_followers_count":566,"pinned_tweet_ids_str":["1508995175024054274"],"possibly_sensitive":false,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1557156541735788544/7wHWKKPD_normal.jpg","profile_interstitial_type":"","screen_name":"Ted04536250","statuses_count":9960,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1670926004184043521"],"editable_until_msecs":"1687218378000","is_edit_eligible":false,"edits_remaining":"5"},"is_translatable":false,"views":{"count":"90","state":"EnabledWithCount"},"source":"Twitter Web App","legacy":{"bookmark_count":0,"bookmarked":false,"created_at":"Mon Jun 19 22:46:18 +0000 2023","conversation_id_str":"1670759701070458880","display_text_range":[13,60],"entities":{"user_mentions":[{"id_str":"1415317224529793028","name":"garfieldbot","screen_name":"cumlasagna1","indices":[0,12]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":1,"favorited":false,"full_text":"@cumlasagna1 He has no voting block for the obvious reasons.","in_reply_to_screen_name":"cumlasagna1","in_reply_to_status_id_str":"1670759701070458880","in_reply_to_user_id_str":"1415317224529793028","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"1499950182842216458","id_str":"1670926004184043521"},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"},"timelinesDetails":{"controllerData":"DAACDAAEDAABCgABFQQCCGAHgAUKAAIAAAAAGADACAAAAAA="}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"conversationthread-1670763351511793664","sortIndex":"7552612335784316897","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1670763351511793664-tweet-1670763351511793664","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1670763351511793664","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDg4NDQzNjE2NTI4MDQ4MTMx","rest_id":"1488443616528048131","affiliates_highlighted_label":{},"is_blue_verified":false,"profile_image_shape":"Circle","legacy":{"created_at":"Tue Feb 01 09:26:35 +0000 2022","default_profile":true,"default_profile_image":false,"description":"What are gems, but coal under pressure.","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":96416,"followers_count":642,"friends_count":372,"has_custom_timelines":true,"is_translator":false,"listed_count":3,"location":"","media_count":2350,"name":"Levi Pendragon","normal_followers_count":642,"pinned_tweet_ids_str":[],"possibly_sensitive":false,"profile_banner_url":"https://pbs.twimg.com/profile_banners/1488443616528048131/1657628435","profile_image_url_https":"https://pbs.twimg.com/profile_images/1609199186897178629/A5R4hdAF_normal.jpg","profile_interstitial_type":"","screen_name":"levi_pendragon","statuses_count":13894,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1670763351511793664"],"editable_until_msecs":"1687179598000","is_edit_eligible":false,"edits_remaining":"5"},"is_translatable":false,"views":{"count":"348","state":"EnabledWithCount"},"source":"Twitter for Android","legacy":{"bookmark_count":0,"bookmarked":false,"created_at":"Mon Jun 19 11:59:58 +0000 2023","conversation_id_str":"1670759701070458880","display_text_range":[13,87],"entities":{"user_mentions":[{"id_str":"1415317224529793028","name":"garfieldbot","screen_name":"cumlasagna1","indices":[0,12]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":9,"favorited":false,"full_text":"@cumlasagna1 He won't support covid lockdowns but will gulag you to stop climate change","in_reply_to_screen_name":"cumlasagna1","in_reply_to_status_id_str":"1670759701070458880","in_reply_to_user_id_str":"1415317224529793028","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":1,"retweet_count":0,"retweeted":false,"user_id_str":"1488443616528048131","id_str":"1670763351511793664"},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"},"timelinesDetails":{"controllerData":"DAACDAAEDAABCgABFQgCCKIDgAUKAAIAAAAACADACAAAAAA="}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"conversationthread-1670858775610241025","sortIndex":"7552612335784316887","content":{"entryType":"TimelineTimelineModule","__typename":"TimelineTimelineModule","items":[{"entryId":"conversationthread-1670858775610241025-tweet-1670858775610241025","item":{"itemContent":{"itemType":"TimelineTweet","__typename":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1670858775610241025","has_birdwatch_notes":false,"core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo4NDk5ODI3OTcxNTQ1OTA3MjA=","rest_id":"849982797154590720","affiliates_highlighted_label":{},"is_blue_verified":false,"profile_image_shape":"Circle","legacy":{"created_at":"Thu Apr 06 13:51:05 +0000 2017","default_profile":true,"default_profile_image":false,"description":"HOMESICK FOR A JOKE THAT MY KIDS ONCE LAUGHED AT","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":531007,"followers_count":7385,"friends_count":486,"has_custom_timelines":true,"is_translator":false,"listed_count":31,"location":"VILLAGE GREEN SOCIETY OFFICE","media_count":386,"name":"THE WORLD'S GREATEST DAD BY WEIGHT AND VOLUME","normal_followers_count":7385,"pinned_tweet_ids_str":["1657070383101509638"],"possibly_sensitive":false,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1472942545093238792/EP_nhOiB_normal.jpg","profile_interstitial_type":"","screen_name":"The_WGD","statuses_count":72995,"translator_type":"none","verified":false,"withheld_in_countries":[]}}}},"edit_control":{"edit_tweet_ids":["1670858775610241025"],"editable_until_msecs":"1687202349000","is_edit_eligible":false,"edits_remaining":"5"},"is_translatable":false,"views":{"count":"251","state":"EnabledWithCount"},"source":"Twitter for Android","legacy":{"bookmark_count":0,"bookmarked":false,"created_at":"Mon Jun 19 18:19:09 +0000 2023","conversation_id_str":"1670759701070458880","display_text_range":[13,55],"entities":{"user_mentions":[{"id_str":"1415317224529793028","name":"garfieldbot","screen_name":"cumlasagna1","indices":[0,12]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":5,"favorited":false,"full_text":"@cumlasagna1 Are you aware that the Kennedys are Irish?","in_reply_to_screen_name":"cumlasagna1","in_reply_to_status_id_str":"1670759701070458880","in_reply_to_user_id_str":"1415317224529793028","is_quote_status":false,"lang":"en","quote_count":0,"reply_count":0,"retweet_count":0,"retweeted":false,"user_id_str":"849982797154590720","id_str":"1670858775610241025"},"quick_promote_eligibility":{"eligibility":"IneligibleUserUnauthorized"}}},"tweetDisplayType":"Tweet"},"clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"},"timelinesDetails":{"controllerData":"DAACDAAEDAABCgABFQQCCSQDgAUKAAIAAAAACADACAAAAAA="}}}}}],"displayType":"VerticalConversation","clientEventInfo":{"details":{"conversationDetails":{"conversationSection":"HighQuality"}}}}},{"entryId":"cursor-showmorethreads-6231663350442984260","sortIndex":"7552612335784316876","content":{"entryType":"TimelineTimelineItem","__typename":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTimelineCursor","__typename":"TimelineTimelineCursor","value":"NwAAAPANHBlWhMC--YvZ3a8ugsDS1aGWqbAugIDQ6dCa3xIAYL314NGqsAkA0OGFzYqwLiUEERUOAAA","cursorType":"ShowMoreThreads","displayTreatment":{"actionText":"Show more replies"}}}}]},{"type":"TimelineTerminateTimeline","direction":"Top"}],"metadata":{"reader_mode_config":{"is_reader_mode_available":true}}}}}