me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tidy up the scraper tests a bit

Browse Source
This commit is contained in:
Alessio 2021-09-27 15:30:05 -07:00
parent f278010201
commit 89179f2447
24 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scraper/api_types_test.go
View File

@ -17,12 +17,13 @@ func TestNormalizeContent(t *testing.T) {
in_reply_to scraper.TweetID
retweeted_status_id scraper.TweetID
} {
{"test_responses/tweet_with_gif_reply.json", "", 0, 1395882872729477131, 0},
{"test_responses/tweet_with_image.json", "this saddens me every time", 0, 0, 0},
{"test_responses/tweet_with_reply.json", "I always liked \"The Anarchist's Cookbook.\"", 0, 1395978577267593218, 0},
{"test_responses/tweet_with_4_images.json", "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do", 0, 0, 0},
{"test_responses/tweet_with_quoted_tweet.json", "", 1422680899670274048, 0, 0},
{"test_responses/tweet_that_is_a_retweet.json", "RT @nofunin10ded: @michaelmalice We're dealing with people who will napalm your children and then laugh about it", 0, 0, 1404269989646028804},
{"test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json", "", 0, 1395882872729477131, 0},
{"test_responses/single_tweets/tweet_with_image.json", "this saddens me every time", 0, 0, 0},
{"test_responses/single_tweets/tweet_that_is_a_reply.json", "Noted", 0, 1396194494710788100, 0},
{"test_responses/single_tweets/tweet_with_4_images.json", "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do", 0, 0, 0},
{"test_responses/single_tweets/tweet_with_quoted_tweet.json", "", 1422680899670274048, 0, 0},
{"test_responses/single_tweets/tweet_with_at_mentions_in_front.json", "It always does, doesn't it?", 0, 1428907275532476416, 0},
{"test_responses/single_tweets/tweet_with_unicode_chars.json", "The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.", 0, 0, 0},
}
for _, v := range test_cases {
data, err := ioutil.ReadFile(v.filename)

2
scraper/image_test.go
View File

@ -9,7 +9,7 @@ import (
)
func TestParseAPIMedia(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/image.json")
data, err := ioutil.ReadFile("test_responses/tweet_content/image.json")
if err != nil {
panic(err)
}

1
scraper/test_responses/single_tweets/tweet_that_is_a_reply.json Normal file
View File

@ -0,0 +1 @@
{"created_at":"Sat May 22 20:33:05 +0000 2021","id_str":"1396202476605296643","full_text":"@RvaTeddy @michaelmalice Noted","display_text_range":[25,30],"entities":{"user_mentions":[{"screen_name":"RvaTeddy","name":"Teddy RVA","id_str":"1215108366411931653","indices":[0,9]},{"screen_name":"michaelmalice","name":"Michael Malice","id_str":"44067298","indices":[10,24]}]},"source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>","in_reply_to_status_id_str":"1396194494710788100","in_reply_to_user_id_str":"1215108366411931653","in_reply_to_screen_name":"RvaTeddy","user_id_str":"15983203","retweet_count":0,"favorite_count":3,"reply_count":0,"quote_count":0,"conversation_id_str":"1395978577267593218","lang":"en"}

1
scraper/test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json Normal file
View File

@ -0,0 +1 @@
{"created_at":"Sat May 22 00:41:18 +0000 2021","id_str":"1395902556707057666","full_text":"@michaelmalice https://t.co/VxuTr4wc5H","display_text_range":[14,14],"entities":{"user_mentions":[{"screen_name":"michaelmalice","name":"Michael Malice","id_str":"44067298","indices":[0,14]}],"media":[{"id_str":"1395902550646300678","indices":[15,38],"media_url":"http://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","media_url_https":"https://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","url":"https://t.co/VxuTr4wc5H","display_url":"pic.twitter.com/VxuTr4wc5H","expanded_url":"https://twitter.com/NaniSuzy/status/1395902556707057666/photo/1","type":"photo","original_info":{"width":498,"height":280},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":498,"h":280,"resize":"fit"},"small":{"w":498,"h":280,"resize":"fit"},"large":{"w":498,"h":280,"resize":"fit"}}}]},"extended_entities":{"media":[{"id_str":"1395902550646300678","indices":[15,38],"media_url":"http://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","media_url_https":"https://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","url":"https://t.co/VxuTr4wc5H","display_url":"pic.twitter.com/VxuTr4wc5H","expanded_url":"https://twitter.com/NaniSuzy/status/1395902556707057666/photo/1","type":"animated_gif","original_info":{"width":498,"height":280},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":498,"h":280,"resize":"fit"},"small":{"w":498,"h":280,"resize":"fit"},"large":{"w":498,"h":280,"resize":"fit"}},"video_info":{"aspect_ratio":[249,140],"variants":[{"bitrate":0,"content_type":"video/mp4","url":"https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4"}]},"media_key":"16_1395902550646300678","ext_alt_text":"The Simpsons GIF by MOODMAN","ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":117,"green":168,"blue":155},"percentage":39.03},{"rgb":{"red":122,"green":111,"blue":109},"percentage":21.2},{"rgb":{"red":167,"green":111,"blue":81},"percentage":13.01},{"rgb":{"red":219,"green":166,"blue":10},"percentage":10.6},{"rgb":{"red":49,"green":99,"blue":12},"percentage":2.92}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>","in_reply_to_status_id_str":"1395882872729477131","in_reply_to_user_id_str":"44067298","in_reply_to_screen_name":"michaelmalice","user_id_str":"1227095561494392832","retweet_count":0,"favorite_count":0,"reply_count":0,"quote_count":0,"conversation_id_str":"1395882872729477131","possibly_sensitive_editable":true,"lang":"und"}

0
scraper/test_responses/tweet_with_4_images.json → scraper/test_responses/single_tweets/tweet_with_4_images.json
View File

1
scraper/test_responses/single_tweets/tweet_with_at_mentions_in_front.json Normal file
View File

@ -0,0 +1 @@
{"created_at":"Sat Aug 21 04:31:59 +0000 2021","id_str":"1428937902877196294","full_text":"@rob_mose @primalpoly @jmasseypoet @SpaceX It always does, doesn't it?","display_text_range":[43,70],"entities":{"user_mentions":[{"screen_name":"rob_mose","name":"Rob 🦇🔊","id_str":"864071849822441472","indices":[0,9]},{"screen_name":"primalpoly","name":"Geoffrey Miller","id_str":"3656836876","indices":[10,21]},{"screen_name":"jmasseypoet","name":"Joseph Massey","id_str":"1107565758652039172","indices":[22,34]},{"screen_name":"SpaceX","name":"SpaceX","id_str":"34743251","indices":[35,42]}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","in_reply_to_status_id_str":"1428907275532476416","in_reply_to_user_id_str":"864071849822441472","in_reply_to_screen_name":"rob_mose","user_id_str":"887434912529338375","retweet_count":0,"favorite_count":2,"reply_count":0,"quote_count":0,"conversation_id_str":"1428807373053911042","lang":"en"}

1
scraper/test_responses/single_tweets/tweet_with_image.json Normal file
View File

@ -0,0 +1 @@
{"created_at":"Fri May 21 23:23:05 +0000 2021","id_str":"1395882872729477131","full_text":"this saddens me every time https://t.co/jSkwGsbKWv","display_text_range":[0,26],"entities":{"media":[{"id_str":"1395882862289772553","indices":[27,50],"media_url":"http://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg","media_url_https":"https://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg","url":"https://t.co/jSkwGsbKWv","display_url":"pic.twitter.com/jSkwGsbKWv","expanded_url":"https://twitter.com/michaelmalice/status/1395882872729477131/photo/1","type":"photo","original_info":{"width":593,"height":239,"focus_rects":[{"x":0,"y":0,"h":239,"w":427},{"x":14,"y":0,"h":239,"w":239},{"x":28,"y":0,"h":239,"w":210},{"x":73,"y":0,"h":239,"w":120},{"x":0,"y":0,"h":239,"w":593}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":593,"h":239,"resize":"fit"},"medium":{"w":593,"h":239,"resize":"fit"},"small":{"w":593,"h":239,"resize":"fit"}}}]},"extended_entities":{"media":[{"id_str":"1395882862289772553","indices":[27,50],"media_url":"http://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg","media_url_https":"https://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg","url":"https://t.co/jSkwGsbKWv","display_url":"pic.twitter.com/jSkwGsbKWv","expanded_url":"https://twitter.com/michaelmalice/status/1395882872729477131/photo/1","type":"photo","original_info":{"width":593,"height":239,"focus_rects":[{"x":0,"y":0,"h":239,"w":427},{"x":14,"y":0,"h":239,"w":239},{"x":28,"y":0,"h":239,"w":210},{"x":73,"y":0,"h":239,"w":120},{"x":0,"y":0,"h":239,"w":593}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":593,"h":239,"resize":"fit"},"medium":{"w":593,"h":239,"resize":"fit"},"small":{"w":593,"h":239,"resize":"fit"}},"media_key":"3_1395882862289772553","ext_alt_text":null,"ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":232,"green":245,"blue":254},"percentage":98.18},{"rgb":{"red":112,"green":127,"blue":140},"percentage":1.44},{"rgb":{"red":109,"green":60,"blue":81},"percentage":0.15},{"rgb":{"red":165,"green":169,"blue":164},"percentage":0.15}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"44067298","retweet_count":3,"favorite_count":374,"reply_count":27,"quote_count":2,"conversation_id_str":"1395882872729477131","possibly_sensitive_editable":true,"lang":"en","self_thread":{"id_str":"1395882872729477131"}}

0
scraper/test_responses/tweet_with_multiple_urls.json → scraper/test_responses/single_tweets/tweet_with_multiple_urls.json
View File

2
scraper/test_responses/tweet_with_quoted_tweet.json → scraper/test_responses/single_tweets/tweet_with_quoted_tweet.json
View File

@ -1 +1 @@
{"created_at":"Tue Aug 03 23:56:40 +0000 2021","id_str":"1422708023768793091","full_text":"https://t.co/8FaOybk9Zz","display_text_range":[0,0],"entities":{"media":[{"id_str":"1422708017481531394","indices":[0,23],"media_url":"http://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","media_url_https":"https://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","url":"https://t.co/8FaOybk9Zz","display_url":"pic.twitter.com/8FaOybk9Zz","expanded_url":"https://twitter.com/michaelmalice/status/1422708023768793091/photo/1","type":"photo","original_info":{"width":2048,"height":1152,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":499,"y":0,"h":1152,"w":1152},{"x":570,"y":0,"h":1152,"w":1011},{"x":787,"y":0,"h":1152,"w":576},{"x":0,"y":0,"h":1152,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":675,"resize":"fit"},"large":{"w":2048,"h":1152,"resize":"fit"},"small":{"w":680,"h":383,"resize":"fit"}}}]},"extended_entities":{"media":[{"id_str":"1422708017481531394","indices":[0,23],"media_url":"http://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","media_url_https":"https://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","url":"https://t.co/8FaOybk9Zz","display_url":"pic.twitter.com/8FaOybk9Zz","expanded_url":"https://twitter.com/michaelmalice/status/1422708023768793091/photo/1","type":"photo","original_info":{"width":2048,"height":1152,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":499,"y":0,"h":1152,"w":1152},{"x":570,"y":0,"h":1152,"w":1011},{"x":787,"y":0,"h":1152,"w":576},{"x":0,"y":0,"h":1152,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":675,"resize":"fit"},"large":{"w":2048,"h":1152,"resize":"fit"},"small":{"w":680,"h":383,"resize":"fit"}},"media_key":"3_1422708017481531394","ext_alt_text":null,"ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":4,"green":0,"blue":0},"percentage":93.11},{"rgb":{"red":85,"green":61,"blue":32},"percentage":2.51},{"rgb":{"red":157,"green":151,"blue":143},"percentage":2.12},{"rgb":{"red":31,"green":33,"blue":45},"percentage":0.68},{"rgb":{"red":79,"green":33,"blue":11},"percentage":0.21}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"44067298","is_quote_status":true,"quoted_status_id_str":"1422680899670274048","quoted_status_permalink":{"url":"https://t.co/6qm3rFL9V0","expanded":"https://twitter.com/steffdaz/status/1422680899670274048","display":"twitter.com/steffdaz/statu…"},"retweet_count":24,"favorite_count":352,"reply_count":9,"quote_count":1,"conversation_id_str":"1422708023768793091","possibly_sensitive_editable":true,"lang":"und"}
{"created_at":"Tue Aug 03 23:56:40 +0000 2021","id_str":"1422708023768793091","full_text":"https://t.co/8FaOybk9Zz","display_text_range":[0,0],"entities":{"media":[{"id_str":"1422708017481531394","indices":[0,23],"media_url":"http://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","media_url_https":"https://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","url":"https://t.co/8FaOybk9Zz","display_url":"pic.twitter.com/8FaOybk9Zz","expanded_url":"https://twitter.com/michaelmalice/status/1422708023768793091/photo/1","type":"photo","original_info":{"width":2048,"height":1152,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":499,"y":0,"h":1152,"w":1152},{"x":570,"y":0,"h":1152,"w":1011},{"x":787,"y":0,"h":1152,"w":576},{"x":0,"y":0,"h":1152,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":675,"resize":"fit"},"large":{"w":2048,"h":1152,"resize":"fit"},"small":{"w":680,"h":383,"resize":"fit"}}}]},"extended_entities":{"media":[{"id_str":"1422708017481531394","indices":[0,23],"media_url":"http://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","media_url_https":"https://pbs.twimg.com/media/E755Z3EWQAIZ0Bf.jpg","url":"https://t.co/8FaOybk9Zz","display_url":"pic.twitter.com/8FaOybk9Zz","expanded_url":"https://twitter.com/michaelmalice/status/1422708023768793091/photo/1","type":"photo","original_info":{"width":2048,"height":1152,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":499,"y":0,"h":1152,"w":1152},{"x":570,"y":0,"h":1152,"w":1011},{"x":787,"y":0,"h":1152,"w":576},{"x":0,"y":0,"h":1152,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":675,"resize":"fit"},"large":{"w":2048,"h":1152,"resize":"fit"},"small":{"w":680,"h":383,"resize":"fit"}},"media_key":"3_1422708017481531394","ext_alt_text":null,"ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":4,"green":0,"blue":0},"percentage":93.11},{"rgb":{"red":85,"green":61,"blue":32},"percentage":2.51},{"rgb":{"red":157,"green":151,"blue":143},"percentage":2.12},{"rgb":{"red":31,"green":33,"blue":45},"percentage":0.68},{"rgb":{"red":79,"green":33,"blue":11},"percentage":0.21}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"44067298","is_quote_status":true,"quoted_status_id_str":"1422680899670274048","quoted_status_permalink":{"url":"https://t.co/6qm3rFL9V0","expanded":"https://twitter.com/steffdaz/status/1422680899670274048","display":"twitter.com/steffdaz/statu…"},"retweet_count":26,"favorite_count":428,"reply_count":7,"quote_count":1,"conversation_id_str":"1422708023768793091","possibly_sensitive_editable":true,"lang":"und"}

1
scraper/test_responses/single_tweets/tweet_with_unicode_chars.json Normal file
View File

@ -0,0 +1 @@
{"created_at":"Fri May 21 23:18:25 +0000 2021","id_str":"1395881699142160387","full_text":"The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.","display_text_range":[0,157],"entities":{"user_mentions":[{"screen_name":"michaelmalice","name":"Michael Malice","id_str":"44067298","indices":[14,28]}]},"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","user_id_str":"467661480","retweet_count":236,"favorite_count":2916,"reply_count":69,"quote_count":14,"conversation_id_str":"1395881699142160387","lang":"en"}

0
scraper/test_responses/tweet_with_url_but_no_card.json → scraper/test_responses/single_tweets/tweet_with_url_but_no_card.json
View File

0
scraper/test_responses/tweet_with_url_card.json → scraper/test_responses/single_tweets/tweet_with_url_card.json
View File

0
scraper/test_responses/tweet_with_video.json → scraper/test_responses/single_tweets/tweet_with_video.json
View File

0
scraper/test_responses/image.json → scraper/test_responses/tweet_content/image.json
View File

0
scraper/test_responses/url_card.json → scraper/test_responses/tweet_content/url_card.json
View File

0
scraper/test_responses/url_card_with_player.json → scraper/test_responses/tweet_content/url_card_with_player.json
View File

0
scraper/test_responses/url_card_without_thumbnail.json → scraper/test_responses/tweet_content/url_card_without_thumbnail.json
View File

0
scraper/test_responses/video.json → scraper/test_responses/tweet_content/video.json
View File

1
scraper/test_responses/tweet_with_gif_reply.json
View File

@ -1 +0,0 @@
{"id_str":"1395902556707057666","conversation_id_str":"1395882872729477131","created_at":"Sat May 22 00:41:18 +0000 2021","favorite_count":0,"full_text":"@michaelmalice https://t.co/VxuTr4wc5H","entities":{"hashtags":null,"media":[{"media_url_https":"https://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","type":"photo","url":"https://t.co/VxuTr4wc5H"}],"urls":null,"Mentions":null},"extended_entities":{"media":[{"id_str":"1395902550646300678","media_url_https":"https://pbs.twimg.com/tweet_video_thumb/E189-VhVoAYcrDv.jpg","type":"animated_gif","video_info":{"variants":[{"url":"https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4"}]}}]},"in_reply_to_status_id_str":"1395882872729477131","in_reply_to_screen_name":"michaelmalice","reply_count":0,"retweet_count":0,"quote_count":0,"retweeted_status_id_str":"","quoted_status_id_str":"","time":"0001-01-01T00:00:00Z","user_id_str":"1227095561494392832"}

1
scraper/test_responses/tweet_with_image.json
View File

@ -1 +0,0 @@
{"created_at":"Fri May 21 23:23:05 +0000 2021","id_str":"1395882872729477131","full_text":"this saddens me every time https:\/\/t.co\/jSkwGsbKWv","display_text_range":[0,26],"entities":{"media":[{"id_str":"1395882862289772553","indices":[27,50],"media_url":"http:\/\/pbs.twimg.com\/media\/E18sEUrWYAk8dBl.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E18sEUrWYAk8dBl.jpg","url":"https:\/\/t.co\/jSkwGsbKWv","display_url":"pic.twitter.com\/jSkwGsbKWv","expanded_url":"https:\/\/twitter.com\/michaelmalice\/status\/1395882872729477131\/photo\/1","type":"photo","original_info":{"width":593,"height":239,"focus_rects":[{"x":0,"y":0,"h":239,"w":427},{"x":14,"y":0,"h":239,"w":239},{"x":28,"y":0,"h":239,"w":210},{"x":73,"y":0,"h":239,"w":120},{"x":0,"y":0,"h":239,"w":593}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":593,"h":239,"resize":"fit"},"medium":{"w":593,"h":239,"resize":"fit"},"small":{"w":593,"h":239,"resize":"fit"}},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}}}]},"extended_entities":{"media":[{"id_str":"1395882862289772553","indices":[27,50],"media_url":"http:\/\/pbs.twimg.com\/media\/E18sEUrWYAk8dBl.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E18sEUrWYAk8dBl.jpg","url":"https:\/\/t.co\/jSkwGsbKWv","display_url":"pic.twitter.com\/jSkwGsbKWv","expanded_url":"https:\/\/twitter.com\/michaelmalice\/status\/1395882872729477131\/photo\/1","type":"photo","original_info":{"width":593,"height":239,"focus_rects":[{"x":0,"y":0,"h":239,"w":427},{"x":14,"y":0,"h":239,"w":239},{"x":28,"y":0,"h":239,"w":210},{"x":73,"y":0,"h":239,"w":120},{"x":0,"y":0,"h":239,"w":593}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":593,"h":239,"resize":"fit"},"medium":{"w":593,"h":239,"resize":"fit"},"small":{"w":593,"h":239,"resize":"fit"}},"features":{"large":{"faces":[]},"medium":{"faces":[]},"small":{"faces":[]},"orig":{"faces":[]}},"media_key":"3_1395882862289772553","ext_alt_text":null,"ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":232,"green":245,"blue":254},"percentage":98.18},{"rgb":{"red":112,"green":127,"blue":140},"percentage":1.44},{"rgb":{"red":109,"green":60,"blue":81},"percentage":0.15},{"rgb":{"red":165,"green":169,"blue":164},"percentage":0.15}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"\u003ca href=\"https:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eTwitter Web App\u003c\/a\u003e","user_id_str":"44067298","retweet_count":4,"favorite_count":317,"reply_count":27,"quote_count":3,"conversation_id_str":"1395882872729477131","possibly_sensitive_editable":true,"lang":"en","self_thread":{"id_str":"1395882872729477131"}}

1
scraper/test_responses/tweet_with_reply.json
View File

@ -1 +0,0 @@
{"id_str":"1396194494710788100","conversation_id_str":"1395978577267593218","created_at":"Sat May 22 20:01:22 +0000 2021","favorite_count":1,"full_text":"@michaelmalice I always liked \"The Anarchist's Cookbook.\"","entities":{"hashtags":null,"media":null,"urls":null,"Mentions":null},"extended_entities":{"media":null},"in_reply_to_status_id_str":"1395978577267593218","in_reply_to_screen_name":"michaelmalice","reply_count":2,"retweet_count":0,"quote_count":1,"retweeted_status_id_str":"","quoted_status_id_str":"","time":"0001-01-01T00:00:00Z","user_id_str":"1215108366411931653"}

8
scraper/tweet_test.go
View File

@ -143,7 +143,7 @@ func TestParseSingleTweet2(t *testing.T) {
func TestParseTweetWithVideo(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/tweet_with_video.json")
data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_video.json")
if err != nil {
panic(err)
}
@ -167,7 +167,7 @@ func TestParseTweetWithVideo(t *testing.T) {
}
func TestParseTweetWithUrl(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/tweet_with_url_card.json")
data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_url_card.json")
if err != nil {
panic(err)
}
@ -199,7 +199,7 @@ func TestParseTweetWithUrl(t *testing.T) {
}
func TestParseTweetWithUrlButNoCard(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/tweet_with_url_but_no_card.json")
data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_url_but_no_card.json")
if err != nil {
panic(err)
}
@ -227,7 +227,7 @@ func TestParseTweetWithUrlButNoCard(t *testing.T) {
}
func TestParseTweetWithMultipleUrls(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/tweet_with_multiple_urls.json")
data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_multiple_urls.json")
if err != nil {
panic(err)
}

6
scraper/url_test.go
View File

@ -9,7 +9,7 @@ import (
)
func TestParseAPIUrlCard(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/url_card.json")
data, err := ioutil.ReadFile("test_responses/tweet_content/url_card.json")
if err != nil {
panic(err)
}
@ -57,7 +57,7 @@ func TestParseAPIUrlCard(t *testing.T) {
}
func TestParseAPIUrlCardWithPlayer(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/url_card_with_player.json")
data, err := ioutil.ReadFile("test_responses/tweet_content/url_card_with_player.json")
if err != nil {
panic(err)
}
@ -101,7 +101,7 @@ func TestParseAPIUrlCardWithPlayer(t *testing.T) {
}
func TestParseAPIUrlCardWithoutThumbnail(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/url_card_without_thumbnail.json")
data, err := ioutil.ReadFile("test_responses/tweet_content/url_card_without_thumbnail.json")
if err != nil {
panic(err)
}

2
scraper/video_test.go
View File

@ -9,7 +9,7 @@ import (
)
func TestParseAPIVideo(t *testing.T) {
data, err := ioutil.ReadFile("test_responses/video.json")
data, err := ioutil.ReadFile("test_responses/tweet_content/video.json")
if err != nil {
panic(err)
}