From 12be70f541800da8488b076ee1c1bc2321d47251 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 30 Jan 2022 17:58:03 -0800 Subject: [PATCH] Add test files --- scraper/test_responses/api_v2/retweet.json | 1 + scraper/test_responses/api_v2/retweet_with_quote_tweet.json | 1 + scraper/test_responses/api_v2/tweet_plaintext.json | 1 + scraper/test_responses/api_v2/tweet_with_quoted_tweet.json | 1 + 4 files changed, 4 insertions(+) create mode 100644 scraper/test_responses/api_v2/retweet.json create mode 100644 scraper/test_responses/api_v2/retweet_with_quote_tweet.json create mode 100644 scraper/test_responses/api_v2/tweet_plaintext.json create mode 100644 scraper/test_responses/api_v2/tweet_with_quoted_tweet.json diff --git a/scraper/test_responses/api_v2/retweet.json b/scraper/test_responses/api_v2/retweet.json new file mode 100644 index 0000000..5d161f1 --- /dev/null +++ b/scraper/test_responses/api_v2/retweet.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485699748514476037","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:43:17 +0000 2022","conversation_id_str":"1485699748514476037","display_text_range":[0,118],"entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[95,118],"media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"photo","url":"https://t.co/3BpvLjdJwD","features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720}}],"user_mentions":[{"id_str":"1326229737551912960","name":"Libs of Tik Tok","screen_name":"libsoftiktok","indices":[3,16]}],"urls":[],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[95,118],"media_key":"7_1485627274594590721","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"video","url":"https://t.co/3BpvLjdJwD","additional_media_info":{"monetizable":false,"source_user":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDI1ODU5MzkzMzA5MDA3ODc3","rest_id":"1425859393309007877","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Aug 12 16:39:17 +0000 2021","default_profile":true,"default_profile_image":false,"description":"👑 DONALD J TRUMP 👑","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":5197,"followers_count":5452,"friends_count":754,"has_custom_timelines":true,"is_translator":false,"listed_count":21,"location":"","media_count":727,"name":"👑 Danie","normal_followers_count":5452,"pinned_tweet_ids_str":["1470122966641950723"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":38.35,"rgb":{"blue":1,"green":1,"red":4}},{"percentage":14.72,"rgb":{"blue":30,"green":63,"red":119}},{"percentage":14.21,"rgb":{"blue":97,"green":143,"red":207}},{"percentage":9.85,"rgb":{"blue":208,"green":216,"red":228}},{"percentage":3.16,"rgb":{"blue":5,"green":8,"red":63}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1425859393309007877/1629816326","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.79,"rgb":{"blue":205,"green":187,"red":174}},{"percentage":12.88,"rgb":{"blue":42,"green":120,"red":165}},{"percentage":11.64,"rgb":{"blue":2,"green":147,"red":140}},{"percentage":6.51,"rgb":{"blue":73,"green":143,"red":230}},{"percentage":5.08,"rgb":{"blue":8,"green":14,"red":40}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1449863641877164032/wJc9WYYM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"1607Danie","statuses_count":5697,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}}},"ext_media_color":{"palette":[{"percentage":28.23,"rgb":{"blue":42,"green":33,"red":36}},{"percentage":21.44,"rgb":{"blue":171,"green":170,"red":158}},{"percentage":12.3,"rgb":{"blue":133,"green":139,"red":188}},{"percentage":9.79,"rgb":{"blue":120,"green":115,"red":85}},{"percentage":6.68,"rgb":{"blue":54,"green":76,"red":108}}]},"mediaStats":{"viewCount":0},"ext_media_availability":{"status":"Available"},"features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720},"video_info":{"aspect_ratio":[1,1],"duration_millis":30066,"variants":[{"bitrate":1280000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/720x720/xxMVzC92dZqILtFh.mp4?tag=12"},{"bitrate":832000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/540x540/N9j_xaaJM82y-JQE.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/pl/TBjt-Zmw7uKX6i_d.m3u8?tag=12&container=fmp4"},{"bitrate":432000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/320x320/L1xYjf9yAVJ1ncz3.mp4?tag=12"}]}}]},"favorite_count":0,"favorited":false,"full_text":"RT @libsoftiktok: More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":0,"reply_count":0,"retweet_count":1305,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485699748514476037","retweeted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485694028620316673","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxMzI2MjI5NzM3NTUxOTEyOTYw","rest_id":"1326229737551912960","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Nov 10 18:26:33 +0000 2020","default_profile":true,"default_profile_image":false,"description":"Independent journalist bringing you the news from tik tok. All videos belong to their respective owners. Inquiries: libsoftiktok@gmail.com. DM submissions","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"linktr.ee/Libsoftiktok","expanded_url":"https://linktr.ee/Libsoftiktok","url":"https://t.co/KqjohqNdCh","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":19708,"followers_count":500837,"friends_count":633,"has_custom_timelines":true,"is_translator":false,"listed_count":1045,"location":"Depths of Hell","media_count":1846,"name":"Libs of Tik Tok","normal_followers_count":500837,"pinned_tweet_ids_str":["1468024511379296256"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":41.04,"rgb":{"blue":243,"green":227,"red":228}},{"percentage":25.16,"rgb":{"blue":2,"green":1,"red":246}},{"percentage":19.93,"rgb":{"blue":105,"green":22,"red":7}},{"percentage":4.25,"rgb":{"blue":2,"green":2,"red":140}},{"percentage":1.98,"rgb":{"blue":83,"green":76,"red":232}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1326229737551912960/1613633975","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":78.81,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":7.39,"rgb":{"blue":248,"green":250,"red":248}},{"percentage":5.41,"rgb":{"blue":80,"green":4,"red":247}},{"percentage":3.56,"rgb":{"blue":231,"green":238,"red":10}},{"percentage":0.62,"rgb":{"blue":13,"green":5,"red":42}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1476706688547778560/cJruRIQ6_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"libsoftiktok","statuses_count":4655,"translator_type":"none","url":"https://t.co/KqjohqNdCh","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:20:33 +0000 2022","conversation_id_str":"1485694028620316673","display_text_range":[0,100],"entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[77,100],"media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"photo","url":"https://t.co/3BpvLjdJwD","features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720}}],"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"extended_entities":{"media":[{"display_url":"pic.twitter.com/3BpvLjdJwD","expanded_url":"https://twitter.com/1607Danie/status/1485627344287109130/video/1","id_str":"1485627274594590721","indices":[77,100],"media_key":"7_1485627274594590721","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1485627274594590721/pu/img/O6mMKrsqWl8WcMy1.jpg","source_status_id_str":"1485627344287109130","source_user_id_str":"1425859393309007877","type":"video","url":"https://t.co/3BpvLjdJwD","additional_media_info":{"monetizable":false,"source_user":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDI1ODU5MzkzMzA5MDA3ODc3","rest_id":"1425859393309007877","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Aug 12 16:39:17 +0000 2021","default_profile":true,"default_profile_image":false,"description":"👑 DONALD J TRUMP 👑","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":5197,"followers_count":5452,"friends_count":754,"has_custom_timelines":true,"is_translator":false,"listed_count":21,"location":"","media_count":727,"name":"👑 Danie","normal_followers_count":5452,"pinned_tweet_ids_str":["1470122966641950723"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":38.35,"rgb":{"blue":1,"green":1,"red":4}},{"percentage":14.72,"rgb":{"blue":30,"green":63,"red":119}},{"percentage":14.21,"rgb":{"blue":97,"green":143,"red":207}},{"percentage":9.85,"rgb":{"blue":208,"green":216,"red":228}},{"percentage":3.16,"rgb":{"blue":5,"green":8,"red":63}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1425859393309007877/1629816326","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.79,"rgb":{"blue":205,"green":187,"red":174}},{"percentage":12.88,"rgb":{"blue":42,"green":120,"red":165}},{"percentage":11.64,"rgb":{"blue":2,"green":147,"red":140}},{"percentage":6.51,"rgb":{"blue":73,"green":143,"red":230}},{"percentage":5.08,"rgb":{"blue":8,"green":14,"red":40}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1449863641877164032/wJc9WYYM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"1607Danie","statuses_count":5697,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}}},"ext_media_color":{"palette":[{"percentage":28.23,"rgb":{"blue":42,"green":33,"red":36}},{"percentage":21.44,"rgb":{"blue":171,"green":170,"red":158}},{"percentage":12.3,"rgb":{"blue":133,"green":139,"red":188}},{"percentage":9.79,"rgb":{"blue":120,"green":115,"red":85}},{"percentage":6.68,"rgb":{"blue":54,"green":76,"red":108}}]},"mediaStats":{"viewCount":10},"ext_media_availability":{"status":"Available"},"features":{},"sizes":{"large":{"h":720,"w":720,"resize":"fit"},"medium":{"h":720,"w":720,"resize":"fit"},"small":{"h":680,"w":680,"resize":"fit"},"thumb":{"h":150,"w":150,"resize":"crop"}},"original_info":{"height":720,"width":720},"video_info":{"aspect_ratio":[1,1],"duration_millis":30066,"variants":[{"bitrate":1280000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/720x720/xxMVzC92dZqILtFh.mp4?tag=12"},{"bitrate":832000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/540x540/N9j_xaaJM82y-JQE.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/pl/TBjt-Zmw7uKX6i_d.m3u8?tag=12&container=fmp4"},{"bitrate":432000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1485627274594590721/pu/vid/320x320/L1xYjf9yAVJ1ncz3.mp4?tag=12"}]}}]},"favorite_count":5373,"favorited":false,"full_text":"More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":776,"reply_count":1010,"retweet_count":1305,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"1326229737551912960","id_str":"1485694028620316673"}}}}}} diff --git a/scraper/test_responses/api_v2/retweet_with_quote_tweet.json b/scraper/test_responses/api_v2/retweet_with_quote_tweet.json new file mode 100644 index 0000000..af3e597 --- /dev/null +++ b/scraper/test_responses/api_v2/retweet_with_quote_tweet.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485273090665984000","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo1OTk4MTczNzg=","rest_id":"599817378","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 05 02:00:29 +0000 2012","default_profile":true,"default_profile_image":false,"description":"Just an Anglo American on Twitter. Highly respected Daily Caller alum. Author of No Campus for White Men. Host of Highly Respected","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"youtube.com/highlyrespected","expanded_url":"https://www.youtube.com/highlyrespected","url":"https://t.co/jtijsC7BOn","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":19019,"followers_count":102575,"friends_count":701,"has_custom_timelines":false,"is_translator":false,"listed_count":666,"location":"","media_count":3268,"name":"Scott Greer 6’2” IQ 187","normal_followers_count":102575,"pinned_tweet_ids_str":["1485699486173306889"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":89.95,"rgb":{"blue":42,"green":33,"red":24}},{"percentage":10.05,"rgb":{"blue":123,"green":115,"red":112}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/599817378/1610662268","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":39.5,"rgb":{"blue":28,"green":16,"red":16}},{"percentage":17.58,"rgb":{"blue":74,"green":28,"red":21}},{"percentage":15.0,"rgb":{"blue":100,"green":109,"red":152}},{"percentage":12.22,"rgb":{"blue":131,"green":10,"red":1}},{"percentage":3.99,"rgb":{"blue":205,"green":203,"red":209}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/922462562075250688/9sYJaPoo_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"ScottMGreer","statuses_count":50681,"translator_type":"none","url":"https://t.co/jtijsC7BOn","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Sun Jan 23 15:27:54 +0000 2022","conversation_id_str":"1485273090665984000","display_text_range":[0,143],"entities":{"user_mentions":[{"id_str":"1434720042193760256","name":"Latino Bodybuilders for Hellenism","screen_name":"LatinxPutler","indices":[3,16]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":0,"favorited":false,"full_text":"RT @LatinxPutler: >Man's lifelong slavery to the Common Good, defined as the arbitrary wishes and claims of just any random lower nobility-…","is_quote_status":true,"lang":"en","quote_count":0,"quoted_status_id_str":"1484900469482962944","quoted_status_permalink":{"url":"https://t.co/vg7WRm6orF","expanded":"https://twitter.com/Heminator/status/1484900469482962944","display":"twitter.com/Heminator/stat…"},"reply_count":0,"retweet_count":16,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"599817378","id_str":"1485273090665984000","retweeted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485272859102621697","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDM0NzIwMDQyMTkzNzYwMjU2","rest_id":"1434720042193760256","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Mon Sep 06 03:29:03 +0000 2021","default_profile":true,"default_profile_image":false,"description":"Gemeinschaft of Latino bodybuilders. Styling on you from our Cartagena villa: fashion, music, militant radical Hellenism and Vladimir Putin's Lebensphilosophie","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":3918,"followers_count":17843,"friends_count":31,"has_custom_timelines":false,"is_translator":false,"listed_count":74,"location":"Cartagena, Colombia","media_count":256,"name":"Latino Bodybuilders for Hellenism","normal_followers_count":17843,"pinned_tweet_ids_str":["1483162645230411777"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":31.47,"rgb":{"blue":5,"green":98,"red":121}},{"percentage":28.56,"rgb":{"blue":5,"green":47,"red":63}},{"percentage":10.65,"rgb":{"blue":163,"green":177,"red":179}},{"percentage":5.4,"rgb":{"blue":5,"green":143,"red":173}},{"percentage":3.54,"rgb":{"blue":6,"green":23,"red":32}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1434720042193760256/1630899846","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":59.7,"rgb":{"blue":219,"green":230,"red":240}},{"percentage":14.37,"rgb":{"blue":76,"green":123,"red":189}},{"percentage":9.27,"rgb":{"blue":11,"green":24,"red":68}},{"percentage":4.23,"rgb":{"blue":23,"green":54,"red":111}},{"percentage":4.05,"rgb":{"blue":8,"green":9,"red":21}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1434722373123682305/sITLBErd_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"LatinxPutler","statuses_count":4325,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quoted_status_result":{"result":{"__typename":"Tweet","rest_id":"1484900469482962944","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxNDM0Nzk3Mg==","rest_id":"14347972","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Thu Apr 10 03:04:38 +0000 2008","default_profile":false,"default_profile_image":false,"description":"Senior Writer at RealClearInvestigations. \"After all these years of professional experience, why can’t I write good?\" Husband of @MZHemingway.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"weeklystandard.com/rough-draft/ar…","expanded_url":"http://www.weeklystandard.com/rough-draft/article/2010315","url":"https://t.co/yrqsduzuVm","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":27218,"followers_count":94652,"friends_count":5677,"has_custom_timelines":false,"is_translator":false,"listed_count":1132,"location":"","media_count":588,"name":"Mark Hemingway","normal_followers_count":94652,"pinned_tweet_ids_str":[],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":83.03,"rgb":{"blue":196,"green":189,"red":180}},{"percentage":9.32,"rgb":{"blue":126,"green":143,"red":156}},{"percentage":6.96,"rgb":{"blue":114,"green":108,"red":103}},{"percentage":0.69,"rgb":{"blue":240,"green":211,"red":191}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/14347972/1532479949","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":68.52,"rgb":{"blue":236,"green":236,"red":236}},{"percentage":17.52,"rgb":{"blue":13,"green":16,"red":251}},{"percentage":4.74,"rgb":{"blue":62,"green":61,"red":59}},{"percentage":2.1,"rgb":{"blue":180,"green":181,"red":250}},{"percentage":2.1,"rgb":{"blue":94,"green":96,"red":248}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/555863013027094528/MUOYcD2g_normal.png","profile_interstitial_type":"","protected":false,"screen_name":"Heminator","statuses_count":17991,"translator_type":"none","url":"https://t.co/yrqsduzuVm","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quotedRefResult":{"result":{"__typename":"Tweet","rest_id":"1484643409130397702"}},"legacy":{"created_at":"Sat Jan 22 14:47:14 +0000 2022","conversation_id_str":"1484900469482962944","display_text_range":[0,82],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":570,"favorited":false,"full_text":"Parental responsibility, much like life, begins at conception. I hope this passes.","is_quote_status":true,"lang":"en","quote_count":2,"quoted_status_id_str":"1484643409130397702","quoted_status_permalink":{"url":"https://t.co/siBqm5HcQ2","expanded":"https://twitter.com/forrestbennett/status/1484643409130397702","display":"twitter.com/forrestbennett…"},"reply_count":13,"retweet_count":65,"retweeted":false,"source":"Twitter for iPhone","user_id_str":"14347972","id_str":"1484900469482962944"}}},"legacy":{"created_at":"Sun Jan 23 15:26:59 +0000 2022","conversation_id_str":"1485272859102621697","display_text_range":[0,274],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":200,"favorited":false,"full_text":">Man's lifelong slavery to the Common Good, defined as the arbitrary wishes and claims of just any random lower nobility--correction, roastie-possessor and her soft-fingered managers in black robes--begins at his conception. This is similar to Medieval Knighthood - trads","is_quote_status":true,"lang":"en","quote_count":2,"quoted_status_id_str":"1484900469482962944","quoted_status_permalink":{"url":"https://t.co/vg7WRm6orF","expanded":"https://twitter.com/Heminator/status/1484900469482962944","display":"twitter.com/Heminator/stat…"},"reply_count":3,"retweet_count":16,"retweeted":false,"source":"Twitter Web App","user_id_str":"1434720042193760256","id_str":"1485272859102621697"}}}}}} diff --git a/scraper/test_responses/api_v2/tweet_plaintext.json b/scraper/test_responses/api_v2/tweet_plaintext.json new file mode 100644 index 0000000..a74b9a5 --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_plaintext.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485708879174508550","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 20:19:34 +0000 2022","conversation_id_str":"1485708879174508550","display_text_range":[0,182],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":38,"favorited":false,"full_text":"If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964","is_quote_status":false,"lang":"en","quote_count":1,"reply_count":2,"retweet_count":2,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485708879174508550"}}} diff --git a/scraper/test_responses/api_v2/tweet_with_quoted_tweet.json b/scraper/test_responses/api_v2/tweet_with_quoted_tweet.json new file mode 100644 index 0000000..6960f3b --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_with_quoted_tweet.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1485690410899021826","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"quoted_status_result":{"result":{"__typename":"Tweet","rest_id":"1485690069079846915","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo4OTIxNTUyMTgyOTI2MTcyMTc=","rest_id":"892155218292617217","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Mon Jul 31 22:49:14 +0000 2017","default_profile":false,"default_profile_image":false,"description":"Creator of Little Homes and Mooncars","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"opensea.io/Baalzimon","expanded_url":"https://opensea.io/Baalzimon","url":"https://t.co/OG8K1chHjG","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":11758,"followers_count":4071,"friends_count":421,"has_custom_timelines":true,"is_translator":false,"listed_count":114,"location":"The Two Rivers","media_count":6395,"name":"Baalzimon","normal_followers_count":4071,"pinned_tweet_ids_str":["1025119632821297154"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":59.57,"rgb":{"blue":255,"green":255,"red":255}},{"percentage":25.47,"rgb":{"blue":107,"green":115,"red":118}},{"percentage":7.5,"rgb":{"blue":158,"green":199,"red":219}},{"percentage":7.19,"rgb":{"blue":186,"green":155,"red":92}},{"percentage":0.25,"rgb":{"blue":197,"green":182,"red":154}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/892155218292617217/1629831304","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.74,"rgb":{"blue":163,"green":163,"red":163}},{"percentage":27.14,"rgb":{"blue":231,"green":189,"red":250}},{"percentage":2.89,"rgb":{"blue":71,"green":68,"red":70}},{"percentage":2.05,"rgb":{"blue":250,"green":250,"red":250}},{"percentage":0.71,"rgb":{"blue":117,"green":86,"red":130}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1441842995913838594/IbbWPT8D_normal.png","profile_interstitial_type":"","protected":false,"screen_name":"baalzimon","statuses_count":38209,"translator_type":"none","url":"https://t.co/OG8K1chHjG","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"legacy":{"created_at":"Mon Jan 24 19:04:49 +0000 2022","conversation_id_str":"1485689207435710464","display_text_range":[15,105],"entities":{"user_mentions":[{"id_str":"44067298","name":"Michael Malice","screen_name":"michaelmalice","indices":[0,14]}],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":12,"favorited":false,"full_text":"@michaelmalice The Left hates the Right so much that they won't let them leave the Union. I don't get it.","in_reply_to_screen_name":"michaelmalice","in_reply_to_status_id_str":"1485689207435710464","in_reply_to_user_id_str":"44067298","is_quote_status":false,"lang":"en","quote_count":1,"reply_count":1,"retweet_count":0,"retweeted":false,"source":"Twitter Web App","user_id_str":"892155218292617217","id_str":"1485690069079846915"}}},"legacy":{"created_at":"Mon Jan 24 19:06:11 +0000 2022","conversation_id_str":"1485690410899021826","display_text_range":[0,125],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":87,"favorited":false,"full_text":"Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, for example","is_quote_status":true,"lang":"en","quote_count":1,"quoted_status_id_str":"1485690069079846915","quoted_status_permalink":{"url":"https://t.co/DoPLoPPD1B","expanded":"https://twitter.com/baalzimon/status/1485690069079846915","display":"twitter.com/baalzimon/stat…"},"reply_count":6,"retweet_count":10,"retweeted":false,"source":"Twitter Web App","user_id_str":"44067298","id_str":"1485690410899021826"}}}