diff --git a/scraper/api_types.go b/scraper/api_types.go index 15eb25f..6c12332 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -163,7 +163,7 @@ type APITweet struct { QuotedStatusIDStr string `json:"quoted_status_id_str"` // Can be empty string QuotedStatusID int64 QuotedStatusPermalink struct { - URL string `json:"url"` + ShortURL string `json:"url"` ExpandedURL string `json:"expanded"` } `json:"quoted_status_permalink"` Time time.Time `json:"time"` @@ -188,10 +188,20 @@ func (t *APITweet) NormalizeContent() { t.FullText = string([]rune(t.FullText)[t.DisplayTextRange[0]:t.DisplayTextRange[1]]) } + // Handle short links showing up at ends of tweets + for _, url := range t.Entities.URLs { + index := strings.Index(t.FullText, url.ShortenedUrl) + if index == (len(t.FullText) - len(url.ShortenedUrl)) { + t.FullText = strings.TrimSpace(t.FullText[0:index]) + + } + } + // Handle pasted tweet links that turn into quote tweets but still have a link in them + // This is a separate case from above because we want it gone even if it's in the middle of the tweet if t.QuotedStatusID != 0 { for _, url := range t.Entities.URLs { - if url.ShortenedUrl == t.QuotedStatusPermalink.URL { + if url.ShortenedUrl == t.QuotedStatusPermalink.ShortURL { t.FullText = strings.ReplaceAll(t.FullText, url.ShortenedUrl, "") } } diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 5e1d10f..e46f8d3 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -10,6 +10,67 @@ import ( "strings" ) +type CardValue struct { + Type string `json:"type"` + StringValue string `json:"string_value"` + ImageValue struct { + AltText string `json:"alt"` + Height int `json:"height"` + Width int `json:"width"` + Url string `json:"url"` + } `json:"image_value"` + UserValue struct { + ID int64 `json:"id_str,string"` + } `json:"user_value"` + BooleanValue bool `json:"boolean_value"` +} + +type APIV2Card struct { + Legacy struct { + BindingValues []struct { + Key string `json:"key"` + Value CardValue `json:"value"` + } `json:"binding_values"` + Name string `json:"name"` + Url string `json:"url"` + } `json:"legacy"` +} +func (card APIV2Card) ParseAsUrl() Url { + values := make(map[string]CardValue) + for _, obj := range card.Legacy.BindingValues { + values[obj.Key] = obj.Value + } + + ret := Url{} + ret.HasCard = true + + ret.ShortText = card.Legacy.Url + ret.Domain = values["domain"].StringValue + ret.Title = values["title"].StringValue + ret.Description = values["description"].StringValue + ret.IsContentDownloaded = false + ret.CreatorID = UserID(values["creator"].UserValue.ID) + ret.SiteID = UserID(values["site"].UserValue.ID) + + var thumbnail_url string + if card.Legacy.Name == "summary_large_image" || card.Legacy.Name == "summary" { + thumbnail_url = values["thumbnail_image_large"].ImageValue.Url + } else if card.Legacy.Name == "player" { + thumbnail_url = values["player_image_large"].ImageValue.Url + } else { + panic("TODO unknown card type") + } + + if thumbnail_url != "" { + ret.HasThumbnail = true + ret.ThumbnailRemoteUrl = thumbnail_url + ret.ThumbnailLocalPath = get_thumbnail_local_path(thumbnail_url) + ret.ThumbnailWidth = values["thumbnail_image_large"].ImageValue.Width + ret.ThumbnailHeight = values["thumbnail_image_large"].ImageValue.Height + } + return ret +} + type APIV2UserResult struct { UserResults struct { Result struct { @@ -37,6 +98,7 @@ type APIV2Result struct { } `json:"text"` } `json:"tombstone"` Core *APIV2UserResult `json:"core"` + Card APIV2Card `json:"card"` QuotedStatusResult *APIV2Result `json:"quoted_status_result"` } `json:"result"` } @@ -74,6 +136,25 @@ func (api_result APIV2Result) ToTweetTrove() TweetTrove { ret.MergeWith(quoted_trove) } + // Handle URL cards + if api_result.Result.Card.Legacy.Name == "summary_large_image" || api_result.Result.Card.Legacy.Name == "player" { + url := api_result.Result.Card.ParseAsUrl() + + main_tweet := ret.Tweets[TweetID(api_result.Result.Legacy.ID)] + found := false + for i := range main_tweet.Urls { + if main_tweet.Urls[i].ShortText != url.ShortText { + continue + } + found = true + url.Text = main_tweet.Urls[i].Text // Copy the expanded URL over, since the card doesn't have it in the new API + main_tweet.Urls[i] = url + } + if !found { + panic("Tweet trove doesn't contain its own main tweet") + } + } + return ret } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index 2012cd3..3ea497f 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -268,6 +268,72 @@ func TestAPIV2ParseRetweetedQuoteTweet(t *testing.T) { } +/** + * Parse a tweet with a link + */ +func TestAPIV2ParseTweetWithURL(t *testing.T) { + assert := assert.New(t) + data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_url.json") + if err != nil { + panic(err) + } + + var tweet_result APIV2Result + err = json.Unmarshal(data, &tweet_result) + assert.NoError(err) + + trove := tweet_result.ToTweetTrove() + + assert.Equal(1, len(trove.Tweets)) + tweet, ok := trove.Tweets[1485695695025803264] + assert.True(ok) + assert.Equal("This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to show they were the Good Guys but also to get the pack to regroup, akin to wolves howling.", tweet.Text) + + assert.Equal(1, len(tweet.Urls)) + url := tweet.Urls[0] + assert.Equal("observer.com", url.Domain) + assert.Equal("Why Evangelical Progressives Need to Demonstrate Anguish Publicly", url.Title) + assert.Equal("https://observer.com/2016/12/why-evangelical-progressives-need-to-demonstrate-anguish-publicly/", url.Text) + assert.Equal("The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to others that one is a good person without having to do anything", url.Description) + assert.Equal("https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x600", url.ThumbnailRemoteUrl) + assert.Equal(600, url.ThumbnailWidth) + assert.Equal(300, url.ThumbnailHeight) + assert.Equal(UserID(15738599), url.SiteID) + assert.Equal(UserID(15738599), url.CreatorID) +} + +/** + * Parse a tweet with a link with a "player" card + */ +func TestAPIV2ParseTweetWithURLPlayerCard(t *testing.T) { + assert := assert.New(t) + data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_url_player_card.json") + if err != nil { + panic(err) + } + + var tweet_result APIV2Result + err = json.Unmarshal(data, &tweet_result) + assert.NoError(err) + + trove := tweet_result.ToTweetTrove() + + assert.Equal(1, len(trove.Tweets)) + tweet, ok := trove.Tweets[1485504913614327808] + assert.True(ok) + assert.Equal("i'll just leave this here", tweet.Text) + + assert.Equal(1, len(tweet.Urls)) + url := tweet.Urls[0] + assert.Equal("www.youtube.com", url.Domain) + assert.Equal("Michael Malice on Kennedy Nov. 15, 2016", url.Title) + assert.Equal("https://www.youtube.com/watch?v=c9TypEM1ik4&t=9s", url.Text) + assert.Equal("Steve Bannon;", url.Description) + assert.Equal("https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=jpg&name=800x320_1", url.ThumbnailRemoteUrl) + assert.Equal(UserID(10228272), url.SiteID) +} + + func TestParseAPIV2UserFeed(t *testing.T) { data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json") if err != nil { diff --git a/scraper/test_responses/api_v2/tweet_with_url.json b/scraper/test_responses/api_v2/tweet_with_url.json new file mode 100644 index 0000000..0fd7b1d --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_with_url.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485695695025803264","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"card":{"rest_id":"https://t.co/UqWeoxRbmH","legacy":{"binding_values":[{"key":"photo_image_full_size_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":419,"width":800,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=800x419"},"type":"IMAGE"}},{"key":"thumbnail_image","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":200,"width":400,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=400x400"},"type":"IMAGE"}},{"key":"description","value":{"string_value":"The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to others that one is a good person without having to do anything","type":"STRING"}},{"key":"domain","value":{"string_value":"observer.com","type":"STRING"}},{"key":"thumbnail_image_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":300,"width":600,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x600"},"type":"IMAGE"}},{"key":"summary_photo_image_small","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":202,"width":386,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=386x202"},"type":"IMAGE"}},{"key":"thumbnail_image_original","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=orig"},"type":"IMAGE"}},{"key":"site","value":{"scribe_key":"publisher_id","type":"USER","user_value":{"id_str":"15738599","path":[]}}},{"key":"photo_image_full_size_small","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":202,"width":386,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=386x202"},"type":"IMAGE"}},{"key":"summary_photo_image_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":419,"width":800,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=800x419"},"type":"IMAGE"}},{"key":"thumbnail_image_small","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":72,"width":144,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=144x144"},"type":"IMAGE"}},{"key":"creator","value":{"type":"USER","user_value":{"id_str":"15738599","path":[]}}},{"key":"thumbnail_image_x_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=png&name=2048x2048_2_exp"},"type":"IMAGE"}},{"key":"photo_image_full_size_original","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=orig"},"type":"IMAGE"}},{"key":"photo_image_full_size_alt_text","value":{"string_value":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","type":"STRING"}},{"key":"vanity_url","value":{"scribe_key":"vanity_url","string_value":"observer.com","type":"STRING"}},{"key":"photo_image_full_size","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":314,"width":600,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x314"},"type":"IMAGE"}},{"key":"summary_photo_image_alt_text","value":{"string_value":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","type":"STRING"}},{"key":"thumbnail_image_color","value":{"image_color_value":{"palette":[{"rgb":{"blue":19,"green":18,"red":16},"percentage":47.66},{"rgb":{"blue":74,"green":91,"red":116},"percentage":18.13},{"rgb":{"blue":126,"green":118,"red":118},"percentage":16.99},{"rgb":{"blue":205,"green":178,"red":163},"percentage":6.96},{"rgb":{"blue":29,"green":24,"red":104},"percentage":3.0}]},"type":"IMAGE_COLOR"}},{"key":"title","value":{"string_value":"Why Evangelical Progressives Need to Demonstrate Anguish Publicly","type":"STRING"}},{"key":"summary_photo_image_color","value":{"image_color_value":{"palette":[{"rgb":{"blue":19,"green":18,"red":16},"percentage":47.66},{"rgb":{"blue":74,"green":91,"red":116},"percentage":18.13},{"rgb":{"blue":126,"green":118,"red":118},"percentage":16.99},{"rgb":{"blue":205,"green":178,"red":163},"percentage":6.96},{"rgb":{"blue":29,"green":24,"red":104},"percentage":3.0}]},"type":"IMAGE_COLOR"}},{"key":"summary_photo_image_x_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=png&name=2048x2048_2_exp"},"type":"IMAGE"}},{"key":"summary_photo_image","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":314,"width":600,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x314"},"type":"IMAGE"}},{"key":"photo_image_full_size_color","value":{"image_color_value":{"palette":[{"rgb":{"blue":19,"green":18,"red":16},"percentage":47.66},{"rgb":{"blue":74,"green":91,"red":116},"percentage":18.13},{"rgb":{"blue":126,"green":118,"red":118},"percentage":16.99},{"rgb":{"blue":205,"green":178,"red":163},"percentage":6.96},{"rgb":{"blue":29,"green":24,"red":104},"percentage":3.0}]},"type":"IMAGE_COLOR"}},{"key":"photo_image_full_size_x_large","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=png&name=2048x2048_2_exp"},"type":"IMAGE"}},{"key":"card_url","value":{"scribe_key":"card_url","string_value":"https://t.co/UqWeoxRbmH","type":"STRING"}},{"key":"summary_photo_image_original","value":{"image_value":{"alt":"A woman protests against US President-elect Donald Trump in front of Trump Tower on November 20, 2016 in New York. / AFP / KENA BETANCUR","height":438,"width":876,"url":"https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=orig"},"type":"IMAGE"}}],"card_platform":{"platform":{"audience":{"name":"production"},"device":{"name":"Swift","version":"12"}}},"name":"summary_large_image","url":"https://t.co/UqWeoxRbmH","user_refs":[{"id":"VXNlcjoxNTczODU5OQ==","rest_id":"15738599","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Aug 05 17:07:20 +0000 2008","default_profile":false,"default_profile_image":false,"description":"New York state of mind.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"observer.com","expanded_url":"http://observer.com","url":"https://t.co/fJtcFwQpsx","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":25467,"followers_count":186128,"friends_count":58738,"has_custom_timelines":true,"is_translator":false,"listed_count":4755,"location":"New York City","media_count":20449,"name":"Observer","normal_followers_count":186128,"pinned_tweet_ids_str":[],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":54.12,"rgb":{"blue":20,"green":26,"red":31}},{"percentage":15.66,"rgb":{"blue":143,"green":168,"red":185}},{"percentage":9.92,"rgb":{"blue":45,"green":83,"red":118}},{"percentage":5.96,"rgb":{"blue":89,"green":153,"red":202}},{"percentage":2.22,"rgb":{"blue":220,"green":211,"red":209}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/15738599/1630895307","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":63.92,"rgb":{"blue":50,"green":50,"red":50}},{"percentage":33.75,"rgb":{"blue":255,"green":255,"red":255}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/879438623531311104/BCOSkIoi_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"observer","statuses_count":94270,"translator_type":"none","url":"https://t.co/fJtcFwQpsx","verified":true,"withheld_in_countries":[]},"professional":{"rest_id":"1463248301986254851","professional_type":"Business"},"super_follow_eligible":false,"super_followed_by":false,"super_following":false},{"id":"VXNlcjoxNTczODU5OQ==","rest_id":"15738599","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Aug 05 17:07:20 +0000 2008","default_profile":false,"default_profile_image":false,"description":"New York state of mind.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"observer.com","expanded_url":"http://observer.com","url":"https://t.co/fJtcFwQpsx","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":25467,"followers_count":186128,"friends_count":58738,"has_custom_timelines":true,"is_translator":false,"listed_count":4755,"location":"New York City","media_count":20449,"name":"Observer","normal_followers_count":186128,"pinned_tweet_ids_str":[],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":54.12,"rgb":{"blue":20,"green":26,"red":31}},{"percentage":15.66,"rgb":{"blue":143,"green":168,"red":185}},{"percentage":9.92,"rgb":{"blue":45,"green":83,"red":118}},{"percentage":5.96,"rgb":{"blue":89,"green":153,"red":202}},{"percentage":2.22,"rgb":{"blue":220,"green":211,"red":209}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/15738599/1630895307","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":63.92,"rgb":{"blue":50,"green":50,"red":50}},{"percentage":33.75,"rgb":{"blue":255,"green":255,"red":255}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/879438623531311104/BCOSkIoi_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"observer","statuses_count":94270,"translator_type":"none","url":"https://t.co/fJtcFwQpsx","verified":true,"withheld_in_countries":[]},"professional":{"rest_id":"1463248301986254851","professional_type":"Business"},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}]}},"legacy":{"created_at":"Mon Jan 24 19:27:11 +0000 2022","conversation_id_str":"1485695693247619072","display_text_range":[0,226],"entities":{"user_mentions":[],"urls":[{"display_url":"observer.com/2016/12/why-ev…","expanded_url":"https://observer.com/2016/12/why-evangelical-progressives-need-to-demonstrate-anguish-publicly/","url":"https://t.co/UqWeoxRbmH","indices":[203,226]}],"hashtags":[],"symbols":[]},"favorite_count":207,"favorited":false,"full_text":"This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to show they were the Good Guys but also to get the pack to regroup, akin to wolves howling.\n\nhttps://t.co/UqWeoxRbmH","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1485695693247619072","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":0,"reply_count":3,"retweet_count":19,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485695695025803264","self_thread":{"id_str":"1485695693247619072"}}}} diff --git a/scraper/test_responses/api_v2/tweet_with_url_player_card.json b/scraper/test_responses/api_v2/tweet_with_url_player_card.json new file mode 100644 index 0000000..d662380 --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_with_url_player_card.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485504913614327808","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"card":{"rest_id":"https://t.co/iPfOi7jHya","legacy":{"binding_values":[{"key":"player_url","value":{"string_value":"https://www.youtube.com/embed/c9TypEM1ik4?start=9","type":"STRING"}},{"key":"player_image_large","value":{"image_value":{"height":320,"width":569,"url":"https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=jpg&name=800x320_1"},"type":"IMAGE"}},{"key":"player_image","value":{"image_value":{"height":158,"width":280,"url":"https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=jpg&name=280x280"},"type":"IMAGE"}},{"key":"app_star_rating","value":{"string_value":"4.68334","type":"STRING"}},{"key":"description","value":{"string_value":"Steve Bannon;","type":"STRING"}},{"key":"player_width","value":{"string_value":"1280","type":"STRING"}},{"key":"domain","value":{"string_value":"www.youtube.com","type":"STRING"}},{"key":"app_is_free","value":{"string_value":"true","type":"STRING"}},{"key":"site","value":{"scribe_key":"publisher_id","type":"USER","user_value":{"id_str":"10228272","path":[]}}},{"key":"player_image_original","value":{"image_value":{"height":720,"width":1280,"url":"https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=jpg&name=orig"},"type":"IMAGE"}},{"key":"app_num_ratings","value":{"string_value":"24,207,204","type":"STRING"}},{"key":"app_price_amount","value":{"string_value":"0.0","type":"STRING"}},{"key":"player_height","value":{"string_value":"720","type":"STRING"}},{"key":"vanity_url","value":{"scribe_key":"vanity_url","string_value":"youtube.com","type":"STRING"}},{"key":"app_name","value":{"string_value":"YouTube: Watch, Listen, Stream","type":"STRING"}},{"key":"player_image_small","value":{"image_value":{"height":81,"width":144,"url":"https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=jpg&name=144x144"},"type":"IMAGE"}},{"key":"title","value":{"string_value":"Michael Malice on Kennedy Nov. 15, 2016","type":"STRING"}},{"key":"app_price_currency","value":{"string_value":"USD","type":"STRING"}},{"key":"card_url","value":{"scribe_key":"card_url","string_value":"https://t.co/iPfOi7jHya","type":"STRING"}},{"key":"player_image_color","value":{"image_color_value":{"palette":[{"rgb":{"blue":23,"green":21,"red":17},"percentage":40.65},{"rgb":{"blue":175,"green":102,"red":42},"percentage":10.97},{"rgb":{"blue":53,"green":18,"red":16},"percentage":9.72},{"rgb":{"blue":30,"green":44,"red":51},"percentage":6.76},{"rgb":{"blue":89,"green":13,"red":38},"percentage":6.13}]},"type":"IMAGE_COLOR"}},{"key":"player_image_x_large","value":{"image_value":{"height":720,"width":1280,"url":"https://pbs.twimg.com/card_img/1485504774233415680/fsbK59th?format=png&name=2048x2048_2_exp"},"type":"IMAGE"}}],"card_platform":{"platform":{"audience":{"name":"production"},"device":{"name":"Swift","version":"12"}}},"name":"player","url":"https://t.co/iPfOi7jHya","user_refs":[{"id":"VXNlcjoxMDIyODI3Mg==","rest_id":"10228272","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Nov 13 21:43:46 +0000 2007","default_profile":false,"default_profile_image":false,"description":"like and subscribe.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"youtube.com","expanded_url":"http://youtube.com","url":"https://t.co/bUisN3Gqbw","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":5953,"followers_count":74097548,"friends_count":1206,"has_custom_timelines":true,"is_translator":false,"listed_count":79783,"location":"San Bruno, CA","media_count":13709,"name":"YouTube","normal_followers_count":74097548,"pinned_tweet_ids_str":[],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":33.47,"rgb":{"blue":219,"green":206,"red":212}},{"percentage":8.85,"rgb":{"blue":230,"green":179,"red":153}},{"percentage":5.41,"rgb":{"blue":141,"green":182,"red":207}},{"percentage":4.36,"rgb":{"blue":185,"green":166,"red":221}},{"percentage":3.42,"rgb":{"blue":188,"green":201,"red":163}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/10228272/1635870163","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":70.88,"rgb":{"blue":255,"green":255,"red":255}},{"percentage":27.69,"rgb":{"blue":0,"green":0,"red":254}},{"percentage":0.86,"rgb":{"blue":178,"green":180,"red":252}},{"percentage":0.37,"rgb":{"blue":90,"green":87,"red":254}},{"percentage":0.15,"rgb":{"blue":130,"green":132,"red":250}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1427292844612595720/RC1YSvuT_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"YouTube","statuses_count":41636,"translator_type":"regular","url":"https://t.co/bUisN3Gqbw","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}]}},"legacy":{"created_at":"Mon Jan 24 06:49:05 +0000 2022","conversation_id_str":"1485500984822489088","display_text_range":[17,67],"entities":{"user_mentions":[{"id_str":"95092020","name":"Dr Jordan B Peterson","screen_name":"jordanbpeterson","indices":[0,16]}],"urls":[{"display_url":"youtube.com/watch?v=c9TypE…","expanded_url":"https://www.youtube.com/watch?v=c9TypEM1ik4&t=9s","url":"https://t.co/iPfOi7jHya","indices":[44,67]}],"hashtags":[],"symbols":[]},"favorite_count":844,"favorited":false,"full_text":"@jordanbpeterson i'll just leave this here\n\nhttps://t.co/iPfOi7jHya","in_reply_to_screen_name":"jordanbpeterson","in_reply_to_status_id_str":"1485500984822489088","in_reply_to_user_id_str":"95092020","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":6,"reply_count":47,"retweet_count":62,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485504913614327808"}}}