From 1bc6aec3b58e523ede3bd9d74188ac4e95462c4f Mon Sep 17 00:00:00 2001 From: Alessio Date: Tue, 1 Feb 2022 16:04:52 -0800 Subject: [PATCH] Incorporate the new parsing structure into the actual program --- scraper/api_types_v2.go | 216 ++------------- scraper/api_types_v2_test.go | 250 ------------------ .../api_v2/feed_quote_tweet.json | 1 - .../api_v2/feed_retweeted_quote_tweet.json | 1 - .../api_v2/feed_simple_retweet.json | 1 - .../api_v2/feed_simple_tweet.json | 1 - 6 files changed, 22 insertions(+), 448 deletions(-) delete mode 100644 scraper/test_responses/api_v2/feed_quote_tweet.json delete mode 100644 scraper/test_responses/api_v2/feed_retweeted_quote_tweet.json delete mode 100644 scraper/test_responses/api_v2/feed_simple_retweet.json delete mode 100644 scraper/test_responses/api_v2/feed_simple_tweet.json diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 9abf02a..5e1d10f 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -114,6 +114,23 @@ func (api_v2_tweet APIV2Tweet) ToTweetTrove() TweetTrove { return ret } +type APIV2Entry struct { + EntryID string `json:"entryId"` + SortIndex int64 `json:"sortIndex,string"` + Content struct { + ItemContent struct { + EntryType string `json:"entryType"` + TweetResults APIV2Result `json:"tweet_results"` + } `json:"itemContent"` + + // Cursors + EntryType string `json:"entryType"` + Value string `json:"value"` + CursorType string `json:"cursorType"` + + } `json:"content"` +} + type APIV2Response struct { Data struct { User struct { @@ -122,108 +139,7 @@ type APIV2Response struct { Timeline struct { Instructions []struct { Type string `json:"type"` - Entries []struct { - EntryID string `json:"entryId"` - SortIndex int64 `json:"sortIndex,string"` - Content struct { - ItemContent struct { - EntryType string `json:"entryType"` - TweetResults struct { - Result struct { - Legacy struct { - APITweet - RetweetedStatusResult struct { - Result struct { - ID int `json:"rest_id,string"` - Legacy APITweet `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - QuotedStatusResult struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APITweet `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - } `json:"result"` - } `json:"quoted_status_result"` - } `json:"result"` - } `json:"retweeted_status_result"` - } `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - QuotedStatusResult struct { // Same as "Result" - Result struct { - ID int64 `json:"rest_id,string"` - Legacy struct { - APITweet - RetweetedStatusResult struct { - Result struct { - ID int `json:"rest_id,string"` - Legacy APITweet `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - QuotedStatusResult struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APITweet `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - } `json:"result"` - } `json:"quoted_status_result"` - } `json:"result"` - } `json:"retweeted_status_result"` - } `json:"legacy"` - Core struct { - UserResults struct { - Result struct { - ID int64 `json:"rest_id,string"` - Legacy APIUser `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - } `json:"result"` - } `json:"quoted_status_result"` - } `json:"result"` - } `json:"tweet_results"` - } `json:"itemContent"` - - // Cursors - EntryType string `json:"entryType"` - Value string `json:"value"` - CursorType string `json:"cursorType"` - - } `json:"content"` - } `json:"entries"` + Entries []APIV2Entry`json:"entries"` } `json:"instructions"` } `json:"timeline"` } `json:"timeline"` @@ -242,7 +158,6 @@ func (api_response APIV2Response) GetCursorBottom() string { return last_entry.Content.Value } - /** * Parse the collected API response and turn it into a TweetTrove */ @@ -250,102 +165,15 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { ret := NewTweetTrove() for _, entry := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries { // TODO: the second Instruction is the pinned tweet if !strings.HasPrefix(entry.EntryID, "tweet-") { - // println(entry.EntryID) continue } - result := entry.Content.ItemContent.TweetResults.Result - apiv2_tweet := result.Legacy - apiv2_user_result := result.Core.UserResults.Result - apiv2_retweeted_tweet_result := apiv2_tweet.RetweetedStatusResult.Result - apiv2_retweeted_tweet_user := apiv2_retweeted_tweet_result.Core.UserResults.Result - apiv2_retweeted_quoted_result := apiv2_retweeted_tweet_result.QuotedStatusResult.Result - apiv2_retweeted_quoted_user := apiv2_retweeted_quoted_result.Core.UserResults.Result - apiv2_quoted_tweet_result := result.QuotedStatusResult.Result - apiv2_quoted_user_result := apiv2_quoted_tweet_result.Core.UserResults.Result + result := entry.Content.ItemContent.TweetResults - // Handle case of retweet (main tweet doesn't get parsed other than retweeted_at) - if apiv2_retweeted_tweet_result.ID != 0 { - orig_tweet, err := ParseSingleTweet(apiv2_retweeted_tweet_result.Legacy) - if err != nil { - return TweetTrove{}, err - } - ret.Tweets[orig_tweet.ID] = orig_tweet - - orig_user, err := ParseSingleUser(apiv2_retweeted_tweet_user.Legacy) - if err != nil { - return TweetTrove{}, err - } - orig_user.ID = UserID(apiv2_retweeted_tweet_user.ID) - ret.Users[orig_user.ID] = orig_user - - retweeting_user, err := ParseSingleUser(apiv2_user_result.Legacy) - if err != nil { - return TweetTrove{}, err - } - retweeting_user.ID = UserID(apiv2_user_result.ID) - ret.Users[retweeting_user.ID] = retweeting_user - - retweet := Retweet{} - retweet.RetweetID = TweetID(apiv2_tweet.ID) - retweet.TweetID = TweetID(orig_tweet.ID) - retweet.RetweetedByID = retweeting_user.ID - retweet.RetweetedAt, err = time.Parse(time.RubyDate, apiv2_tweet.CreatedAt) - if err != nil { - fmt.Printf("%v\n", apiv2_tweet) - panic(err) - } - ret.Retweets[retweet.RetweetID] = retweet - - // Handle quoted tweet - if apiv2_retweeted_quoted_result.ID != 0 { - quoted_tweet, err := ParseSingleTweet(apiv2_retweeted_quoted_result.Legacy) - if err != nil { - return TweetTrove{}, err - } - ret.Tweets[quoted_tweet.ID] = quoted_tweet - - quoted_user, err := ParseSingleUser(apiv2_retweeted_quoted_user.Legacy) - if err != nil { - return TweetTrove{}, err - } - quoted_user.ID = UserID(apiv2_retweeted_quoted_user.ID) - ret.Users[quoted_user.ID] = quoted_user - } - - continue - } - - // The main tweet - tweet, err := ParseSingleTweet(apiv2_tweet.APITweet) - if err != nil { - return TweetTrove{}, err - } - ret.Tweets[tweet.ID] = tweet - - user, err := ParseSingleUser(apiv2_user_result.Legacy) - if err != nil { - return TweetTrove{}, err - } - user.ID = UserID(apiv2_user_result.ID) - ret.Users[user.ID] = user - - // Handle quoted tweet - if apiv2_quoted_tweet_result.ID != 0 { - quoted_tweet, err := ParseSingleTweet(apiv2_quoted_tweet_result.Legacy.APITweet) - if err != nil { - return TweetTrove{}, err - } - ret.Tweets[quoted_tweet.ID] = quoted_tweet - - quoted_user, err := ParseSingleUser(apiv2_quoted_user_result.Legacy) - if err != nil { - return TweetTrove{}, err - } - quoted_user.ID = UserID(apiv2_quoted_user_result.ID) - ret.Users[quoted_user.ID] = quoted_user - } + main_trove := result.ToTweetTrove() + ret.MergeWith(main_trove) } + return ret, nil } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index e2657b7..2012cd3 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -47,7 +47,6 @@ func TestAPIV2ParseUser(t *testing.T) { assert.Equal(user.PinnedTweetID, TweetID(1477347403023982596)) } - /** * Parse a plain text tweet */ @@ -96,7 +95,6 @@ func TestAPIV2ParseTweet(t *testing.T) { assert.Equal(0, len(trove.Retweets)) } - /** * Parse a tweet with a quoted tweet */ @@ -150,7 +148,6 @@ func TestAPIV2ParseTweetWithQuotedTweet(t *testing.T) { assert.Equal(0, len(trove.Retweets)) } - /** * Parse a retweet */ @@ -211,7 +208,6 @@ func TestAPIV2ParseRetweet(t *testing.T) { assert.Equal(UserID(44067298), retweet.RetweetedByID) } - /** * Parse a retweeted quote tweet */ @@ -272,252 +268,6 @@ func TestAPIV2ParseRetweetedQuoteTweet(t *testing.T) { } -// Check a plain old tweet -func TestAPIV2FeedSimpleTweet(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/feed_simple_tweet.json") - if err != nil { - panic(err) - } - var feed APIV2Response - err = json.Unmarshal(data, &feed) - if err != nil { - t.Errorf(err.Error()) - } - - tweet_trove, err := feed.ToTweetTrove() - if err != nil { - panic(err) - } - - if len(tweet_trove.Users) != 1 { - t.Errorf("Expected 1 user, got %d", len(tweet_trove.Users)) - } - user := tweet_trove.Users[44067298] - if user.ID != 44067298 { - t.Errorf("Expected ID %d, got %d", 44067298, user.ID) - } - if user.DisplayName != "Michael Malice" { - t.Errorf("Expected display name %q, got %q", "Michael Malice", user.DisplayName) - } - - - if len(tweet_trove.Tweets) != 1 { - t.Errorf("Expected %d tweets, got %d", 1, len(tweet_trove.Tweets)) - } - tweet := tweet_trove.Tweets[1485708879174508550] - if tweet.ID != 1485708879174508550 { - t.Errorf("Expected ID 1485708879174508550, got %d", tweet.ID) - } - if tweet.UserID != UserID(44067298) { - t.Errorf("Expected user ID 44067298, got %d", tweet.UserID) - } - expected_text := "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964" - if tweet.Text != expected_text { - t.Errorf("Expected text: %q, got: %q", expected_text, tweet.Text) - } - - if len(tweet_trove.Retweets) != 0 { - t.Errorf("Shouldn't be any retweets") - } -} - - -// Check a retweet -func TestAPIV2FeedRetweet(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/feed_simple_retweet.json") - if err != nil { - panic(err) - } - var feed APIV2Response - err = json.Unmarshal(data, &feed) - if err != nil { - t.Errorf(err.Error()) - } - - tweet_trove, err := feed.ToTweetTrove() - if err != nil { - panic(err) - } - - // Should fetch both the retweeting and retweeted users - if len(tweet_trove.Users) != 2 { - t.Errorf("Expected %d users, got %d", 2, len(tweet_trove.Users)) - } - user := tweet_trove.Users[44067298] - if user.ID != 44067298 { - t.Errorf("Expected ID %d, got %d", 44067298, user.ID) - } - if user.DisplayName != "Michael Malice" { - t.Errorf("Expected display name %q, got %q", "Michael Malice", user.DisplayName) - } - - retweeted_user := tweet_trove.Users[1326229737551912960] - if retweeted_user.ID != 1326229737551912960 { - t.Errorf("Expected ID %d, got %d", 1326229737551912960, retweeted_user.ID) - } - if retweeted_user.Handle != "libsoftiktok" { - t.Errorf("Expected handle %q, got %q", "libsoftiktok", retweeted_user.Handle) - } - - // Should only be 1 tweet, the retweeted one - if len(tweet_trove.Tweets) != 1 { - t.Errorf("Expected %d tweets, got %d", 1, len(tweet_trove.Tweets)) - } - tweet, ok := tweet_trove.Tweets[1485694028620316673] - if !ok { - t.Fatalf("Didn't get the tweet") - } - if tweet.ID != 1485694028620316673 { - t.Errorf("Expected ID %d, got %d", 1485694028620316673, tweet.ID) - } - if tweet.UserID != UserID(1326229737551912960) { - t.Errorf("Expected user ID %d, got %d", 1326229737551912960, tweet.UserID) - } - expected_text := "More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD" - if tweet.Text != expected_text { - t.Errorf("Expected text: %q, got: %q", expected_text, tweet.Text) - } - - // Should be 1 retweet - if len(tweet_trove.Retweets) != 1 { - t.Errorf("Expected %d retweets, got %d", 1, len(tweet_trove.Retweets)) - } - retweet := tweet_trove.Retweets[1485699748514476037] - if retweet.RetweetID != 1485699748514476037 { - t.Errorf("Expected RetweetID %d, got %d", 1485699748514476037, retweet.RetweetID) - } - if retweet.TweetID != 1485694028620316673 { - t.Errorf("Expected TweetID 1485694028620316673, got %d", retweet.TweetID) - } - if retweet.RetweetedAt.Unix() != 1643053397 { - t.Errorf("Expected retweeted_at %d, got %d", 1643053397, retweet.RetweetedAt.Unix()) - } - if retweet.RetweetedByID != UserID(44067298) { - t.Errorf("Expected retweeted_by 44067298, got %d", retweet.RetweetedByID) - } -} - - -// Check a quote-tweet -func TestAPIV2FeedQuoteTweet(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/feed_quote_tweet.json") - if err != nil { - panic(err) - } - var feed APIV2Response - err = json.Unmarshal(data, &feed) - if err != nil { - t.Errorf(err.Error()) - } - - tweet_trove, err := feed.ToTweetTrove() - if err != nil { - panic(err) - } - - // Should be 2 users: quoter and quoted - if len(tweet_trove.Users) != 2 { - t.Errorf("Expected %d users, got %d", 2, len(tweet_trove.Users)) - } - quoting_user := tweet_trove.Users[44067298] - if quoting_user.ID != 44067298 { - t.Errorf("Expected quoting user ID %d, got %d", 44067298, quoting_user.ID) - } - quoted_user := tweet_trove.Users[892155218292617217] - if quoted_user.ID != 892155218292617217 { - t.Errorf("Expected quoted user ID %d, got %d", 892155218292617217, quoted_user.ID) - } - expected_quoted_bio := "Creator of Little Homes and Mooncars" - if quoted_user.Bio != expected_quoted_bio { - t.Errorf("Expected bio %q, got %q", expected_quoted_bio, quoted_user.Bio) - } - - - // Should be 2 tweets: quote-tweet and quoted-tweet - if len(tweet_trove.Tweets) != 2 { - t.Errorf("Expected %d tweets, got %d", 2, len(tweet_trove.Tweets)) - } - quoted_tweet := tweet_trove.Tweets[1485690069079846915] - if quoted_tweet.ID != 1485690069079846915 { - t.Errorf("Expected quoted ID %d, got %d", 1485690069079846915, quoted_tweet.ID) - } - expected_quoted_text := "The Left hates the Right so much that they won't let them leave the Union. I don't get it." - if quoted_tweet.Text != expected_quoted_text { - t.Errorf("Expected text %q, got %q", expected_quoted_text, quoted_tweet.Text) - } - quote_tweet := tweet_trove.Tweets[1485690410899021826] - if quote_tweet.ID != 1485690410899021826 { - t.Errorf("Expected quoting ID %d, got %d", 1485690410899021826, quote_tweet.ID) - } - if quote_tweet.QuotedTweetID != 1485690069079846915 { - t.Errorf("Expected to be quoting tweet ID %d, got %d", 1485690069079846915, quote_tweet.QuotedTweetID) - } - - - // No retweets - if len(tweet_trove.Retweets) != 0 { - t.Errorf("Shouldn't be any retweets") - } -} - - -// Check a retweeted quote-tweet -func TestAPIV2FeedRetweetedQuoteTweet(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/feed_retweeted_quote_tweet.json") - if err != nil { - panic(err) - } - var feed APIV2Response - err = json.Unmarshal(data, &feed) - if err != nil { - t.Errorf(err.Error()) - } - - tweet_trove, err := feed.ToTweetTrove() - if err != nil { - panic(err) - } - - // 3 Users: quoted, quoter, and retweeter - if len(tweet_trove.Users) != 3 { - t.Errorf("Expected %d users, got %d", 3, len(tweet_trove.Users)) - } - retweeting_user := tweet_trove.Users[599817378] - if retweeting_user.ID != 599817378 { - t.Errorf("Expected retweeting user ID %d, got %d", 599817378, retweeting_user.ID) - } - if retweeting_user.Website != "https://www.youtube.com/highlyrespected" { - t.Errorf("Expected RTing user website %q, got %q", "https://www.youtube.com/highlyrespected", retweeting_user.Website) - } - retweeted_user := tweet_trove.Users[1434720042193760256] - if retweeted_user.ID != 1434720042193760256 { - t.Errorf("Expected retweed user ID %d, got %d", 1434720042193760256, retweeted_user.ID) - } - if retweeted_user.FollowersCount != 17843 { - t.Errorf("Expected %d followers, got %d", 17843, retweeted_user.FollowersCount) - } - quoted_user := tweet_trove.Users[14347972] - if quoted_user.ID != 14347972 { - t.Errorf("Expected quoted user ID %d, got %d", 14347972, quoted_user.ID) - } - if quoted_user.IsVerified != true { - t.Errorf("Expected quoted user to be verified") - } - - - // Quoted tweet and quoting tweet - if len(tweet_trove.Tweets) != 2 { - t.Errorf("Expected %d tweets, got %d", 2, len(tweet_trove.Tweets)) - } - - // The retweet - if len(tweet_trove.Retweets) != 1 { - t.Errorf("Expected %d retweets, got %d", 1, len(tweet_trove.Retweets)) - } -} - - - func TestParseAPIV2UserFeed(t *testing.T) { data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json") if err != nil { diff --git a/scraper/test_responses/api_v2/feed_quote_tweet.json b/scraper/test_responses/api_v2/feed_quote_tweet.json deleted file mode 100644 index 6e2a0b7..0000000 --- a/scraper/test_responses/api_v2/feed_quote_tweet.json +++ /dev/null @@ -1 +0,0 @@ -{"data":{"user":{"result":{"__typename":"User","timeline":{"timeline":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1485690410899021826","sortIndex":"1485690410899021826","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1485690410899021826","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quoted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485690069079846915","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo4OTIxNTUyMTgyOTI2MTcyMTc=","rest_id":"892155218292617217","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Mon Jul 31 22:49:14 +0000 2017","default_profile":false,"default_profile_image":false,"description":"Creator of Little Homes and Mooncars","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"opensea.io/Baalzimon","expanded_url":"https://opensea.io/Baalzimon","url":"https://t.co/OG8K1chHjG","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":11758,"followers_count":4071,"friends_count":421,"has_custom_timelines":true,"is_translator":false,"listed_count":114,"location":"The Two Rivers","media_count":6395,"name":"Baalzimon","normal_followers_count":4071,"pinned_tweet_ids_str":["1025119632821297154"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":59.57,"rgb":{"blue":255,"green":255,"red":255}},{"percentage":25.47,"rgb":{"blue":107,"green":115,"red":118}},{"percentage":7.5,"rgb":{"blue":158,"green":199,"red":219}},{"percentage":7.19,"rgb":{"blue":186,"green":155,"red":92}},{"percentage":0.25,"rgb":{"blue":197,"green":182,"red":154}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/892155218292617217/1629831304","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.74,"rgb":{"blue":163,"green":163,"red":163}},{"percentage":27.14,"rgb":{"blue":231,"green":189,"red":250}},{"percentage":2.89,"rgb":{"blue":71,"green":68,"red":70}},{"percentage":2.05,"rgb":{"blue":250,"green":250,"red":250}},{"percentage":0.71,"rgb":{"blue":117,"green":86,"red":130}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1441842995913838594/IbbWPT8D_normal.png","profile_interstitial_type":"","protected":false,"screen_name":"baalzimon","statuses_count":38209,"translator_type":"none","url":"https://t.co/OG8K1chHjG","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:04:49 +0000 2022","conversation_id_str":"1485689207435710464","display_text_range":[15,105],"entities":{"user_mentions":[{"id_str":"44067298","name":"Michael Malice","screen_name":"michaelmalice","indices":[0,14]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":12,"favorited":false,"full_text":"@michaelmalice The Left hates the Right so much that they won't let them leave the Union. I don't get it.","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1485689207435710464","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","quote_count":1,"reply_count":1,"retweet_count":0,"retweeted":false,"source":"Twitter Web App","user_id_str":"892155218292617217","id_str":"1485690069079846915"}}},"legacy":{"created_at":"Mon Jan 24 19:06:11 +0000 2022","conversation_id_str":"1485690410899021826","display_text_range":[0,125],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":87,"favorited":false,"full_text":"Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, for example","is_quote_status":true,"lang":"en","quote_count":1,"quoted_status_id_str":"1485690069079846915","quoted_status_permalink":{"url":"https://t.co/DoPLoPPD1B","expanded":"https://twitter.com/baalzimon/status/1485690069079846915","display":"twitter.com/baalzimon/stat…"},"reply_count":6,"retweet_count":10,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485690410899021826"}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWhIC5vdTjnZ4pAAAA"}}}]},{"type":"TimelinePinEntry","entry":{"entryId":"tweet-1477347403023982596","sortIndex":"1485710641850220544","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1477347403023982596","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sat Jan 01 18:34:03 +0000 2022","conversation_id_str":"1469757271341256705","display_text_range":[0,76],"entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}],"user_mentions":[],"urls":[{"display_url":"AnarchistAudiobook.com","expanded_url":"http://AnarchistAudiobook.com","url":"https://t.co/6lCDVUPz05","indices":[53,76]}],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_key":"3_1477347346367369228","media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","ext_media_color":{"palette":[{"percentage":47.83,"rgb":{"blue":13,"green":13,"red":21}},{"percentage":14.09,"rgb":{"blue":56,"green":88,"red":126}},{"percentage":6.88,"rgb":{"blue":183,"green":223,"red":252}},{"percentage":5.64,"rgb":{"blue":109,"green":69,"red":240}},{"percentage":4.09,"rgb":{"blue":47,"green":21,"red":34}}]},"ext_media_availability":{"status":"Available"},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}]},"favorite_count":856,"favorited":false,"full_text":"THE GRAND FINALE\n\nAn all-star cast, available now at https://t.co/6lCDVUPz05 https://t.co/ra6b4YhsDI","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1476988732880347140","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":8,"reply_count":63,"retweet_count":62,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1477347403023982596","self_thread":{"id_str":"1469757271341256705"}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWiIC58dfoy4ApAAAA","socialContext":{"type":"TimelineGeneralContext","contextType":"Pin","text":"Pinned Tweet"}},"clientEventInfo":{"component":"suggest_pinned_tweet","details":{"timelinesDetails":{"injectionType":"PinnedTweet"}}}}}}]}}}}}} diff --git a/scraper/test_responses/api_v2/feed_retweeted_quote_tweet.json b/scraper/test_responses/api_v2/feed_retweeted_quote_tweet.json deleted file mode 100644 index 69804b7..0000000 --- a/scraper/test_responses/api_v2/feed_retweeted_quote_tweet.json +++ /dev/null @@ -1 +0,0 @@ -{"data":{"user":{"result":{"__typename":"User","timeline":{"timeline":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1485273090665984000","sortIndex":"1485273090665984000","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1485273090665984000","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo1OTk4MTczNzg=","rest_id":"599817378","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 05 02:00:29 +0000 2012","default_profile":true,"default_profile_image":false,"description":"Just an Anglo American on Twitter. Highly respected Daily Caller alum. Author of No Campus for White Men. Host of Highly Respected","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"youtube.com/highlyrespected","expanded_url":"https://www.youtube.com/highlyrespected","url":"https://t.co/jtijsC7BOn","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":19019,"followers_count":102575,"friends_count":701,"has_custom_timelines":false,"is_translator":false,"listed_count":666,"location":"","media_count":3268,"name":"Scott Greer 6’2” IQ 187","normal_followers_count":102575,"pinned_tweet_ids_str":["1485699486173306889"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":89.95,"rgb":{"blue":42,"green":33,"red":24}},{"percentage":10.05,"rgb":{"blue":123,"green":115,"red":112}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/599817378/1610662268","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.5,"rgb":{"blue":28,"green":16,"red":16}},{"percentage":17.58,"rgb":{"blue":74,"green":28,"red":21}},{"percentage":15.0,"rgb":{"blue":100,"green":109,"red":152}},{"percentage":12.22,"rgb":{"blue":131,"green":10,"red":1}},{"percentage":3.99,"rgb":{"blue":205,"green":203,"red":209}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/922462562075250688/9sYJaPoo_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"ScottMGreer","statuses_count":50681,"translator_type":"none","url":"https://t.co/jtijsC7BOn","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sun Jan 23 15:27:54 +0000 2022","conversation_id_str":"1485273090665984000","display_text_range":[0,143],"entities":{"user_mentions":[{"id_str":"1434720042193760256","name":"Latino Bodybuilders for Hellenism","screen_name":"LatinxPutler","indices":[3,16]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":0,"favorited":false,"full_text":"RT @LatinxPutler: >Man's lifelong slavery to the Common Good, defined as the arbitrary wishes and claims of just any random lower nobility-…","is_quote_status":true,"lang":"en","quote_count":0,"quoted_status_id_str":"1484900469482962944","quoted_status_permalink":{"url":"https://t.co/vg7WRm6orF","expanded":"https://twitter.com/Heminator/status/1484900469482962944","display":"twitter.com/Heminator/stat…"},"reply_count":0,"retweet_count":16,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"599817378","id_str":"1485273090665984000","retweeted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485272859102621697","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDM0NzIwMDQyMTkzNzYwMjU2","rest_id":"1434720042193760256","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Mon Sep 06 03:29:03 +0000 2021","default_profile":true,"default_profile_image":false,"description":"Gemeinschaft of Latino bodybuilders. Styling on you from our Cartagena villa: fashion, music, militant radical Hellenism and Vladimir Putin's Lebensphilosophie","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":3918,"followers_count":17843,"friends_count":31,"has_custom_timelines":false,"is_translator":false,"listed_count":74,"location":"Cartagena, Colombia","media_count":256,"name":"Latino Bodybuilders for Hellenism","normal_followers_count":17843,"pinned_tweet_ids_str":["1483162645230411777"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":31.47,"rgb":{"blue":5,"green":98,"red":121}},{"percentage":28.56,"rgb":{"blue":5,"green":47,"red":63}},{"percentage":10.65,"rgb":{"blue":163,"green":177,"red":179}},{"percentage":5.4,"rgb":{"blue":5,"green":143,"red":173}},{"percentage":3.54,"rgb":{"blue":6,"green":23,"red":32}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1434720042193760256/1630899846","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":59.7,"rgb":{"blue":219,"green":230,"red":240}},{"percentage":14.37,"rgb":{"blue":76,"green":123,"red":189}},{"percentage":9.27,"rgb":{"blue":11,"green":24,"red":68}},{"percentage":4.23,"rgb":{"blue":23,"green":54,"red":111}},{"percentage":4.05,"rgb":{"blue":8,"green":9,"red":21}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1434722373123682305/sITLBErd_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"LatinxPutler","statuses_count":4325,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quoted_status_result":{"result":{"__typename":"Tweet","rest_id":"1484900469482962944","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDM0Nzk3Mg==","rest_id":"14347972","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Apr 10 03:04:38 +0000 2008","default_profile":false,"default_profile_image":false,"description":"Senior Writer at RealClearInvestigations. \"After all these years of professional experience, why can’t I write good?\" Husband of @MZHemingway.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"weeklystandard.com/rough-draft/ar…","expanded_url":"http://www.weeklystandard.com/rough-draft/article/2010315","url":"https://t.co/yrqsduzuVm","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":27218,"followers_count":94652,"friends_count":5677,"has_custom_timelines":false,"is_translator":false,"listed_count":1132,"location":"","media_count":588,"name":"Mark Hemingway","normal_followers_count":94652,"pinned_tweet_ids_str":[],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":83.03,"rgb":{"blue":196,"green":189,"red":180}},{"percentage":9.32,"rgb":{"blue":126,"green":143,"red":156}},{"percentage":6.96,"rgb":{"blue":114,"green":108,"red":103}},{"percentage":0.69,"rgb":{"blue":240,"green":211,"red":191}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/14347972/1532479949","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":68.52,"rgb":{"blue":236,"green":236,"red":236}},{"percentage":17.52,"rgb":{"blue":13,"green":16,"red":251}},{"percentage":4.74,"rgb":{"blue":62,"green":61,"red":59}},{"percentage":2.1,"rgb":{"blue":180,"green":181,"red":250}},{"percentage":2.1,"rgb":{"blue":94,"green":96,"red":248}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/555863013027094528/MUOYcD2g_normal.png","profile_interstitial_type":"","protected":false,"screen_name":"Heminator","statuses_count":17991,"translator_type":"none","url":"https://t.co/yrqsduzuVm","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quotedRefResult":{"result":{"__typename":"Tweet","rest_id":"1484643409130397702"}},"legacy":{"created_at":"Sat Jan 22 14:47:14 +0000 2022","conversation_id_str":"1484900469482962944","display_text_range":[0,82],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":570,"favorited":false,"full_text":"Parental responsibility, much like life, begins at conception. I hope this passes.","is_quote_status":true,"lang":"en","quote_count":2,"quoted_status_id_str":"1484643409130397702","quoted_status_permalink":{"url":"https://t.co/siBqm5HcQ2","expanded":"https://twitter.com/forrestbennett/status/1484643409130397702","display":"twitter.com/forrestbennett…"},"reply_count":13,"retweet_count":65,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"14347972","id_str":"1484900469482962944"}}},"legacy":{"created_at":"Sun Jan 23 15:26:59 +0000 2022","conversation_id_str":"1485272859102621697","display_text_range":[0,274],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":200,"favorited":false,"full_text":">Man's lifelong slavery to the Common Good, defined as the arbitrary wishes and claims of just any random lower nobility--correction, roastie-possessor and her soft-fingered managers in black robes--begins at his conception. This is similar to Medieval Knighthood - trads","is_quote_status":true,"lang":"en","quote_count":2,"quoted_status_id_str":"1484900469482962944","quoted_status_permalink":{"url":"https://t.co/vg7WRm6orF","expanded":"https://twitter.com/Heminator/status/1484900469482962944","display":"twitter.com/Heminator/stat…"},"reply_count":3,"retweet_count":16,"retweeted":false,"source":"Twitter Web App","user_id_str":"1434720042193760256","id_str":"1485272859102621697"}}}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWgsC5lfny35wpAAAA"}}}]},{"type":"TimelinePinEntry","entry":{"entryId":"tweet-1477347403023982596","sortIndex":"1485710641850220544","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1477347403023982596","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sat Jan 01 18:34:03 +0000 2022","conversation_id_str":"1469757271341256705","display_text_range":[0,76],"entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}],"user_mentions":[],"urls":[{"display_url":"AnarchistAudiobook.com","expanded_url":"http://AnarchistAudiobook.com","url":"https://t.co/6lCDVUPz05","indices":[53,76]}],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_key":"3_1477347346367369228","media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","ext_media_color":{"palette":[{"percentage":47.83,"rgb":{"blue":13,"green":13,"red":21}},{"percentage":14.09,"rgb":{"blue":56,"green":88,"red":126}},{"percentage":6.88,"rgb":{"blue":183,"green":223,"red":252}},{"percentage":5.64,"rgb":{"blue":109,"green":69,"red":240}},{"percentage":4.09,"rgb":{"blue":47,"green":21,"red":34}}]},"ext_media_availability":{"status":"Available"},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}]},"favorite_count":856,"favorited":false,"full_text":"THE GRAND FINALE\n\nAn all-star cast, available now at https://t.co/6lCDVUPz05 https://t.co/ra6b4YhsDI","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1476988732880347140","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":8,"reply_count":63,"retweet_count":62,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1477347403023982596","self_thread":{"id_str":"1469757271341256705"}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWiIC58dfoy4ApAAAA","socialContext":{"type":"TimelineGeneralContext","contextType":"Pin","text":"Pinned Tweet"}},"clientEventInfo":{"component":"suggest_pinned_tweet","details":{"timelinesDetails":{"injectionType":"PinnedTweet"}}}}}}]}}}}}} diff --git a/scraper/test_responses/api_v2/feed_simple_retweet.json b/scraper/test_responses/api_v2/feed_simple_retweet.json deleted file mode 100644 index 8662c23..0000000 --- a/scraper/test_responses/api_v2/feed_simple_retweet.json +++ /dev/null @@ -1 +0,0 @@ -{"data":{"user":{"result":{"__typename":"User","timeline":{"timeline":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1485699748514476037","sortIndex":"1485699748514476037","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1485699748514476037","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:43:17 +0000 2022","conversation_id_str":"1485699748514476037","display_text_range":[0,118],"entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[95,118],"media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"photo","url":"https://t.co/3BpvLjdJwD","features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720}}],"user_mentions":[{"id_str":"1326229737551912960","name":"Libs of Tik Tok","screen_name":"libsoftiktok","indices":[3,16]}],"urls":[],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[95,118],"media_key":"7_1485627274594590721","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"video","url":"https://t.co/3BpvLjdJwD","additional_media_info":{"monetizable":false,"source_user":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDI1ODU5MzkzMzA5MDA3ODc3","rest_id":"1425859393309007877","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Aug 12 16:39:17 +0000 2021","default_profile":true,"default_profile_image":false,"description":"👑 DONALD J TRUMP 👑","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":5197,"followers_count":5452,"friends_count":754,"has_custom_timelines":true,"is_translator":false,"listed_count":21,"location":"","media_count":727,"name":"👑 Danie","normal_followers_count":5452,"pinned_tweet_ids_str":["1470122966641950723"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":38.35,"rgb":{"blue":1,"green":1,"red":4}},{"percentage":14.72,"rgb":{"blue":30,"green":63,"red":119}},{"percentage":14.21,"rgb":{"blue":97,"green":143,"red":207}},{"percentage":9.85,"rgb":{"blue":208,"green":216,"red":228}},{"percentage":3.16,"rgb":{"blue":5,"green":8,"red":63}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1425859393309007877/1629816326","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.79,"rgb":{"blue":205,"green":187,"red":174}},{"percentage":12.88,"rgb":{"blue":42,"green":120,"red":165}},{"percentage":11.64,"rgb":{"blue":2,"green":147,"red":140}},{"percentage":6.51,"rgb":{"blue":73,"green":143,"red":230}},{"percentage":5.08,"rgb":{"blue":8,"green":14,"red":40}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1449863641877164032/wJc9WYYM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"1607Danie","statuses_count":5697,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}}},"ext_media_color":{"palette":[{"percentage":28.23,"rgb":{"blue":42,"green":33,"red":36}},{"percentage":21.44,"rgb":{"blue":171,"green":170,"red":158}},{"percentage":12.3,"rgb":{"blue":133,"green":139,"red":188}},{"percentage":9.79,"rgb":{"blue":120,"green":115,"red":85}},{"percentage":6.68,"rgb":{"blue":54,"green":76,"red":108}}]},"mediaStats":{"viewCount":0},"ext_media_availability":{"status":"Available"},"features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720},"video_info":{"aspect_ratio":[1,1],"duration_millis":30066,"variants":[{"bitrate":1280000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/720x720/xxMVzC92dZqILtFh.mp4?tag=12"},{"bitrate":832000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/540x540/N9j_xaaJM82y-JQE.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/pl/TBjt-Zmw7uKX6i_d.m3u8?tag=12&container=fmp4"},{"bitrate":432000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/320x320/L1xYjf9yAVJ1ncz3.mp4?tag=12"}]}}]},"favorite_count":0,"favorited":false,"full_text":"RT @libsoftiktok: More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":0,"reply_count":0,"retweet_count":1305,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485699748514476037","retweeted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485694028620316673","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxMzI2MjI5NzM3NTUxOTEyOTYw","rest_id":"1326229737551912960","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Nov 10 18:26:33 +0000 2020","default_profile":true,"default_profile_image":false,"description":"Independent journalist bringing you the news from tik tok. All videos belong to their respective owners. Inquiries: libsoftiktok@gmail.com. DM submissions","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"linktr.ee/Libsoftiktok","expanded_url":"https://linktr.ee/Libsoftiktok","url":"https://t.co/KqjohqNdCh","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":19708,"followers_count":500837,"friends_count":633,"has_custom_timelines":true,"is_translator":false,"listed_count":1045,"location":"Depths of Hell","media_count":1846,"name":"Libs of Tik Tok","normal_followers_count":500837,"pinned_tweet_ids_str":["1468024511379296256"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":41.04,"rgb":{"blue":243,"green":227,"red":228}},{"percentage":25.16,"rgb":{"blue":2,"green":1,"red":246}},{"percentage":19.93,"rgb":{"blue":105,"green":22,"red":7}},{"percentage":4.25,"rgb":{"blue":2,"green":2,"red":140}},{"percentage":1.98,"rgb":{"blue":83,"green":76,"red":232}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1326229737551912960/1613633975","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":78.81,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":7.39,"rgb":{"blue":248,"green":250,"red":248}},{"percentage":5.41,"rgb":{"blue":80,"green":4,"red":247}},{"percentage":3.56,"rgb":{"blue":231,"green":238,"red":10}},{"percentage":0.62,"rgb":{"blue":13,"green":5,"red":42}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1476706688547778560/cJruRIQ6_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"libsoftiktok","statuses_count":4655,"translator_type":"none","url":"https://t.co/KqjohqNdCh","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:20:33 +0000 2022","conversation_id_str":"1485694028620316673","display_text_range":[0,100],"entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[77,100],"media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"photo","url":"https://t.co/3BpvLjdJwD","features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720}}],"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[77,100],"media_key":"7_1485627274594590721","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"video","url":"https://t.co/3BpvLjdJwD","additional_media_info":{"monetizable":false,"source_user":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDI1ODU5MzkzMzA5MDA3ODc3","rest_id":"1425859393309007877","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Aug 12 16:39:17 +0000 2021","default_profile":true,"default_profile_image":false,"description":"👑 DONALD J TRUMP 👑","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":5197,"followers_count":5452,"friends_count":754,"has_custom_timelines":true,"is_translator":false,"listed_count":21,"location":"","media_count":727,"name":"👑 Danie","normal_followers_count":5452,"pinned_tweet_ids_str":["1470122966641950723"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":38.35,"rgb":{"blue":1,"green":1,"red":4}},{"percentage":14.72,"rgb":{"blue":30,"green":63,"red":119}},{"percentage":14.21,"rgb":{"blue":97,"green":143,"red":207}},{"percentage":9.85,"rgb":{"blue":208,"green":216,"red":228}},{"percentage":3.16,"rgb":{"blue":5,"green":8,"red":63}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1425859393309007877/1629816326","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.79,"rgb":{"blue":205,"green":187,"red":174}},{"percentage":12.88,"rgb":{"blue":42,"green":120,"red":165}},{"percentage":11.64,"rgb":{"blue":2,"green":147,"red":140}},{"percentage":6.51,"rgb":{"blue":73,"green":143,"red":230}},{"percentage":5.08,"rgb":{"blue":8,"green":14,"red":40}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1449863641877164032/wJc9WYYM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"1607Danie","statuses_count":5697,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}}},"ext_media_color":{"palette":[{"percentage":28.23,"rgb":{"blue":42,"green":33,"red":36}},{"percentage":21.44,"rgb":{"blue":171,"green":170,"red":158}},{"percentage":12.3,"rgb":{"blue":133,"green":139,"red":188}},{"percentage":9.79,"rgb":{"blue":120,"green":115,"red":85}},{"percentage":6.68,"rgb":{"blue":54,"green":76,"red":108}}]},"mediaStats":{"viewCount":0},"ext_media_availability":{"status":"Available"},"features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720},"video_info":{"aspect_ratio":[1,1],"duration_millis":30066,"variants":[{"bitrate":1280000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/720x720/xxMVzC92dZqILtFh.mp4?tag=12"},{"bitrate":832000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/540x540/N9j_xaaJM82y-JQE.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/pl/TBjt-Zmw7uKX6i_d.m3u8?tag=12&container=fmp4"},{"bitrate":432000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/320x320/L1xYjf9yAVJ1ncz3.mp4?tag=12"}]}}]},"favorite_count":5373,"favorited":false,"full_text":"More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":776,"reply_count":1010,"retweet_count":1305,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"1326229737551912960","id_str":"1485694028620316673"}}}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWgoCqzZ62n54pAAAA"}}}]},{"type":"TimelinePinEntry","entry":{"entryId":"tweet-1477347403023982596","sortIndex":"1485710641850220544","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1477347403023982596","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sat Jan 01 18:34:03 +0000 2022","conversation_id_str":"1469757271341256705","display_text_range":[0,76],"entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}],"user_mentions":[],"urls":[{"display_url":"AnarchistAudiobook.com","expanded_url":"http://AnarchistAudiobook.com","url":"https://t.co/6lCDVUPz05","indices":[53,76]}],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_key":"3_1477347346367369228","media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","ext_media_color":{"palette":[{"percentage":47.83,"rgb":{"blue":13,"green":13,"red":21}},{"percentage":14.09,"rgb":{"blue":56,"green":88,"red":126}},{"percentage":6.88,"rgb":{"blue":183,"green":223,"red":252}},{"percentage":5.64,"rgb":{"blue":109,"green":69,"red":240}},{"percentage":4.09,"rgb":{"blue":47,"green":21,"red":34}}]},"ext_media_availability":{"status":"Available"},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}]},"favorite_count":856,"favorited":false,"full_text":"THE GRAND FINALE\n\nAn all-star cast, available now at https://t.co/6lCDVUPz05 https://t.co/ra6b4YhsDI","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1476988732880347140","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":8,"reply_count":63,"retweet_count":62,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1477347403023982596","self_thread":{"id_str":"1469757271341256705"}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWiIC58dfoy4ApAAAA","socialContext":{"type":"TimelineGeneralContext","contextType":"Pin","text":"Pinned Tweet"}},"clientEventInfo":{"component":"suggest_pinned_tweet","details":{"timelinesDetails":{"injectionType":"PinnedTweet"}}}}}}]}}}}}} diff --git a/scraper/test_responses/api_v2/feed_simple_tweet.json b/scraper/test_responses/api_v2/feed_simple_tweet.json deleted file mode 100644 index e9703bb..0000000 --- a/scraper/test_responses/api_v2/feed_simple_tweet.json +++ /dev/null @@ -1 +0,0 @@ -{"data":{"user":{"result":{"__typename":"User","timeline":{"timeline":{"instructions":[{"type":"TimelineAddEntries","entries":[{"entryId":"tweet-1485708879174508550","sortIndex":"1485708879174508550","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1485708879174508550","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 20:19:34 +0000 2022","conversation_id_str":"1485708879174508550","display_text_range":[0,182],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":38,"favorited":false,"full_text":"If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964","is_quote_status":false,"lang":"en","quote_count":1,"reply_count":2,"retweet_count":2,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485708879174508550"}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWjIC57dOWpp4pAAAA"}}}]},{"type":"TimelinePinEntry","entry":{"entryId":"tweet-1477347403023982596","sortIndex":"1485710641850220544","content":{"entryType":"TimelineTimelineItem","itemContent":{"itemType":"TimelineTweet","tweet_results":{"result":{"__typename":"Tweet","rest_id":"1477347403023982596","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sat Jan 01 18:34:03 +0000 2022","conversation_id_str":"1469757271341256705","display_text_range":[0,76],"entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}],"user_mentions":[],"urls":[{"display_url":"AnarchistAudiobook.com","expanded_url":"http://AnarchistAudiobook.com","url":"https://t.co/6lCDVUPz05","indices":[53,76]}],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/ra6b4YhsDI","expanded_url":"https://twitter.com/michaelmalice/status/1477347403023982596/photo/1","id_str":"1477347346367369228","indices":[77,100],"media_key":"3_1477347346367369228","media_url_https":"https://pbs.twimg.com/media/FICXlY4X0AwyB2m.jpg","type":"photo","url":"https://t.co/ra6b4YhsDI","ext_media_color":{"palette":[{"percentage":47.83,"rgb":{"blue":13,"green":13,"red":21}},{"percentage":14.09,"rgb":{"blue":56,"green":88,"red":126}},{"percentage":6.88,"rgb":{"blue":183,"green":223,"red":252}},{"percentage":5.64,"rgb":{"blue":109,"green":69,"red":240}},{"percentage":4.09,"rgb":{"blue":47,"green":21,"red":34}}]},"ext_media_availability":{"status":"Available"},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"sizes":{"large":{"h":900,"w":1200,"resize":"fit"},"medium":{"h":900,"w":1200,"resize":"fit"},"small":{"h":510,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":900,"width":1200,"focus_rects":[{"x":0,"y":0,"w":1200,"h":672},{"x":0,"y":0,"w":900,"h":900},{"x":0,"y":0,"w":789,"h":900},{"x":105,"y":0,"w":450,"h":900},{"x":0,"y":0,"w":1200,"h":900}]}}]},"favorite_count":856,"favorited":false,"full_text":"THE GRAND FINALE\n\nAn all-star cast, available now at https://t.co/6lCDVUPz05 https://t.co/ra6b4YhsDI","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1476988732880347140","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":8,"reply_count":63,"retweet_count":62,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1477347403023982596","self_thread":{"id_str":"1469757271341256705"}}}},"tweetDisplayType":"Tweet","ruxContext":"HHwWiIC58dfoy4ApAAAA","socialContext":{"type":"TimelineGeneralContext","contextType":"Pin","text":"Pinned Tweet"}},"clientEventInfo":{"component":"suggest_pinned_tweet","details":{"timelinesDetails":{"injectionType":"PinnedTweet"}}}}}}]}}}}}}