diff --git a/.build.yml b/.build.yml index 352d519..eeafa56 100644 --- a/.build.yml +++ b/.build.yml @@ -71,8 +71,8 @@ tasks: - install_golangci-lint: | SECONDS=0 - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/go/bin v1.40.1 - sudo ln /usr/local/go/bin/golangci-lint /usr/bin/golangci-lint + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/go/bin v1.50.0 + sudo ln /usr/local/go/bin/golangci-lint /usr/local/bin/golangci-lint duration=$SECONDS echo "Task completed in $(($duration / 60))m$(($duration % 60))s." diff --git a/scraper/test_responses/single_tweets/tweet_with_2_videos.json b/scraper/test_responses/single_tweets/tweet_with_2_videos.json new file mode 100644 index 0000000..88e0241 --- /dev/null +++ b/scraper/test_responses/single_tweets/tweet_with_2_videos.json @@ -0,0 +1,499 @@ +{ + "created_at": "Tue Oct 11 05:21:28 +0000 2022", + "id_str": "1579703711450173440", + "full_text": "RT @FairwayWill: This is what we mean when we say to homeschool your kids. https://t.co/nAPgzhnlz5", + "display_text_range": + [ + 0, + 98 + ], + "entities": + { + "user_mentions": + [ + { + "screen_name": "FairwayWill", + "name": "VrilliamA30🌲", + "id_str": "1574496532732297219", + "indices": + [ + 3, + 15 + ] + } + ], + "media": + [ + { + "id_str": "1579701730148847617", + "indices": + [ + 75, + 98 + ], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1579701730148847617/pu/img/xUlghaCXbPOVN7vI.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1579701730148847617/pu/img/xUlghaCXbPOVN7vI.jpg", + "url": "https://t.co/nAPgzhnlz5", + "display_url": "pic.twitter.com/nAPgzhnlz5", + "expanded_url": "https://twitter.com/FairwayWill/status/1579701786252042240/video/1", + "type": "photo", + "original_info": + { + "width": 576, + "height": 576 + }, + "sizes": + { + "medium": + { + "w": 576, + "h": 576, + "resize": "fit" + }, + "thumb": + { + "w": 150, + "h": 150, + "resize": "crop" + }, + "small": + { + "w": 576, + "h": 576, + "resize": "fit" + }, + "large": + { + "w": 576, + "h": 576, + "resize": "fit" + } + }, + "source_status_id_str": "1579701786252042240", + "source_user_id_str": "1574496532732297219" + }, + { + "id_str": "1579701730157252608", + "indices": + [ + 75, + 98 + ], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1579701730157252608/pu/img/dYN55HDytKvM1Bi8.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1579701730157252608/pu/img/dYN55HDytKvM1Bi8.jpg", + "url": "https://t.co/nAPgzhnlz5", + "display_url": "pic.twitter.com/nAPgzhnlz5", + "expanded_url": "https://twitter.com/FairwayWill/status/1579701786252042240/video/1", + "type": "photo", + "original_info": + { + "width": 480, + "height": 480 + }, + "sizes": + { + "small": + { + "w": 480, + "h": 480, + "resize": "fit" + }, + "thumb": + { + "w": 150, + "h": 150, + "resize": "crop" + }, + "medium": + { + "w": 480, + "h": 480, + "resize": "fit" + }, + "large": + { + "w": 480, + "h": 480, + "resize": "fit" + } + }, + "source_status_id_str": "1579701786252042240", + "source_user_id_str": "1574496532732297219" + } + ] + }, + "extended_entities": + { + "media": + [ + { + "id_str": "1579701730148847617", + "indices": + [ + 75, + 98 + ], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1579701730148847617/pu/img/xUlghaCXbPOVN7vI.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1579701730148847617/pu/img/xUlghaCXbPOVN7vI.jpg", + "url": "https://t.co/nAPgzhnlz5", + "display_url": "pic.twitter.com/nAPgzhnlz5", + "expanded_url": "https://twitter.com/FairwayWill/status/1579701786252042240/video/1", + "type": "video", + "original_info": + { + "width": 576, + "height": 576 + }, + "sizes": + { + "medium": + { + "w": 576, + "h": 576, + "resize": "fit" + }, + "thumb": + { + "w": 150, + "h": 150, + "resize": "crop" + }, + "small": + { + "w": 576, + "h": 576, + "resize": "fit" + }, + "large": + { + "w": 576, + "h": 576, + "resize": "fit" + } + }, + "source_status_id_str": "1579701786252042240", + "source_user_id_str": "1574496532732297219", + "video_info": + { + "aspect_ratio": + [ + 1, + 1 + ], + "duration_millis": 22000, + "variants": + [ + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1579701730148847617/pu/vid/576x576/ghA0fyf58v-2naWR.mp4?tag=12" + }, + { + "bitrate": 432000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1579701730148847617/pu/vid/320x320/671dYD0ejGlOSd6R.mp4?tag=12" + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/ext_tw_video/1579701730148847617/pu/pl/RhrZ9R8L5--gSKsL.m3u8?tag=12&container=fmp4" + } + ] + }, + "media_key": "7_1579701730148847617", + "ext_media_availability": + { + "status": "available" + }, + "ext_alt_text": null, + "ext": + { + "mediaStats": + { + "r": + { + "ok": + { + "viewCount": "49904" + } + }, + "ttl": -1 + } + }, + "additional_media_info": + { + "monetizable": false, + "source_user": + { + "id_str": "1574496532732297219", + "name": "VrilliamA30🌲", + "screen_name": "FairwayWill", + "location": "", + "description": "π”»π• π•˜ β„π•šπ•˜π•™π•₯ β€’ 𝔾𝕠𝕝𝕗 β„π•šπ•˜π•™π•₯", + "entities": + { + "description": + {} + }, + "followers_count": 423, + "fast_followers_count": 0, + "normal_followers_count": 423, + "friends_count": 131, + "listed_count": 1, + "created_at": "Mon Sep 26 20:30:20 +0000 2022", + "favourites_count": 1285, + "statuses_count": 740, + "media_count": 394, + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1574496961742487555/ZolVP9XN_normal.jpg", + "profile_banner_url": "https://pbs.twimg.com/profile_banners/1574496532732297219/1664225370", + "profile_image_extensions_media_availability": null, + "profile_image_extensions_alt_text": null, + "profile_image_extensions": + { + "mediaStats": + { + "r": + { + "missing": null + }, + "ttl": -1 + } + }, + "profile_banner_extensions_media_availability": null, + "profile_banner_extensions_alt_text": null, + "profile_banner_extensions": + { + "mediaStats": + { + "r": + { + "missing": null + }, + "ttl": -1 + } + }, + "profile_link_color": "1DA1F2", + "has_extended_profile": true, + "default_profile": true, + "pinned_tweet_ids": + [ + 1576305061835476992 + ], + "pinned_tweet_ids_str": + [ + "1576305061835476992" + ], + "advertiser_account_type": "none", + "advertiser_account_service_levels": + [], + "profile_interstitial_type": "", + "business_profile_state": "none", + "translator_type": "none", + "withheld_in_countries": + [], + "ext": + { + "highlightedLabel": + { + "r": + { + "ok": + {} + }, + "ttl": -1 + } + } + } + } + }, + { + "id_str": "1579701730157252608", + "indices": + [ + 75, + 98 + ], + "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1579701730157252608/pu/img/dYN55HDytKvM1Bi8.jpg", + "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1579701730157252608/pu/img/dYN55HDytKvM1Bi8.jpg", + "url": "https://t.co/nAPgzhnlz5", + "display_url": "pic.twitter.com/nAPgzhnlz5", + "expanded_url": "https://twitter.com/FairwayWill/status/1579701786252042240/video/1", + "type": "video", + "original_info": + { + "width": 480, + "height": 480 + }, + "sizes": + { + "small": + { + "w": 480, + "h": 480, + "resize": "fit" + }, + "thumb": + { + "w": 150, + "h": 150, + "resize": "crop" + }, + "medium": + { + "w": 480, + "h": 480, + "resize": "fit" + }, + "large": + { + "w": 480, + "h": 480, + "resize": "fit" + } + }, + "source_status_id_str": "1579701786252042240", + "source_user_id_str": "1574496532732297219", + "video_info": + { + "aspect_ratio": + [ + 1, + 1 + ], + "duration_millis": 32934, + "variants": + [ + { + "bitrate": 832000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1579701730157252608/pu/vid/480x480/VQ69Ut84XT2BgIzX.mp4?tag=12" + }, + { + "bitrate": 432000, + "content_type": "video/mp4", + "url": "https://video.twimg.com/ext_tw_video/1579701730157252608/pu/vid/320x320/bB0WzeheFR57X3Yc.mp4?tag=12" + }, + { + "content_type": "application/x-mpegURL", + "url": "https://video.twimg.com/ext_tw_video/1579701730157252608/pu/pl/Frf80trymdNz_TFc.m3u8?tag=12&container=fmp4" + } + ] + }, + "media_key": "7_1579701730157252608", + "ext_media_availability": + { + "status": "available" + }, + "ext_alt_text": null, + "ext": + { + "mediaStats": + { + "r": + { + "ok": + { + "viewCount": "16026" + } + }, + "ttl": -1 + } + }, + "additional_media_info": + { + "monetizable": false, + "source_user": + { + "id_str": "1574496532732297219", + "name": "VrilliamA30🌲", + "screen_name": "FairwayWill", + "location": "", + "description": "π”»π• π•˜ β„π•šπ•˜π•™π•₯ β€’ 𝔾𝕠𝕝𝕗 β„π•šπ•˜π•™π•₯", + "entities": + { + "description": + {} + }, + "followers_count": 423, + "fast_followers_count": 0, + "normal_followers_count": 423, + "friends_count": 131, + "listed_count": 1, + "created_at": "Mon Sep 26 20:30:20 +0000 2022", + "favourites_count": 1285, + "statuses_count": 740, + "media_count": 394, + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1574496961742487555/ZolVP9XN_normal.jpg", + "profile_banner_url": "https://pbs.twimg.com/profile_banners/1574496532732297219/1664225370", + "profile_image_extensions_media_availability": null, + "profile_image_extensions_alt_text": null, + "profile_image_extensions": + { + "mediaStats": + { + "r": + { + "missing": null + }, + "ttl": -1 + } + }, + "profile_banner_extensions_media_availability": null, + "profile_banner_extensions_alt_text": null, + "profile_banner_extensions": + { + "mediaStats": + { + "r": + { + "missing": null + }, + "ttl": -1 + } + }, + "profile_link_color": "1DA1F2", + "has_extended_profile": true, + "default_profile": true, + "pinned_tweet_ids": + [ + 1576305061835476992 + ], + "pinned_tweet_ids_str": + [ + "1576305061835476992" + ], + "advertiser_account_type": "none", + "advertiser_account_service_levels": + [], + "profile_interstitial_type": "", + "business_profile_state": "none", + "translator_type": "none", + "withheld_in_countries": + [], + "ext": + { + "highlightedLabel": + { + "r": + { + "ok": + {} + }, + "ttl": -1 + } + } + } + } + } + ] + }, + "source": "Twitter for Android", + "user_id_str": "1025410166483832833", + "retweeted_status_id_str": "1579701786252042240", + "retweet_count": 351, + "favorite_count": 0, + "reply_count": 0, + "quote_count": 0, + "conversation_id_str": "1579703711450173440", + "possibly_sensitive_editable": true, + "lang": "en" +} diff --git a/scraper/test_responses/single_tweets/tweet_with_image_and_video.json b/scraper/test_responses/single_tweets/tweet_with_image_and_video.json new file mode 100644 index 0000000..00571ee --- /dev/null +++ b/scraper/test_responses/single_tweets/tweet_with_image_and_video.json @@ -0,0 +1 @@ +{"created_at":"Mon Oct 10 02:06:36 +0000 2022","id_str":"1579292281898766336","full_text":"https://t.co/VYKgE22wYb","display_text_range":[0,0],"entities":{"media":[{"id_str":"1579292192580911104","indices":[0,23],"media_url":"http://pbs.twimg.com/media/FerF4bdVQAAKeYJ.jpg","media_url_https":"https://pbs.twimg.com/media/FerF4bdVQAAKeYJ.jpg","url":"https://t.co/VYKgE22wYb","display_url":"pic.twitter.com/VYKgE22wYb","expanded_url":"https://twitter.com/mexicanwilddog/status/1579292281898766336/video/1","type":"photo","original_info":{"width":2048,"height":1279,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":640,"y":0,"h":1279,"w":1279},{"x":718,"y":0,"h":1279,"w":1122},{"x":959,"y":0,"h":1279,"w":640},{"x":0,"y":0,"h":1279,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":749,"resize":"fit"},"large":{"w":2048,"h":1279,"resize":"fit"},"small":{"w":680,"h":425,"resize":"fit"}}},{"id_str":"1579292197752430592","indices":[0,23],"media_url":"http://pbs.twimg.com/ext_tw_video_thumb/1579292197752430592/pu/img/soG4wMWOy3AVpllM.jpg","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1579292197752430592/pu/img/soG4wMWOy3AVpllM.jpg","url":"https://t.co/VYKgE22wYb","display_url":"pic.twitter.com/VYKgE22wYb","expanded_url":"https://twitter.com/mexicanwilddog/status/1579292281898766336/video/1","type":"photo","original_info":{"width":640,"height":750},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":640,"h":750,"resize":"fit"},"small":{"w":580,"h":680,"resize":"fit"},"large":{"w":640,"h":750,"resize":"fit"}}}]},"extended_entities":{"media":[{"id_str":"1579292192580911104","indices":[0,23],"media_url":"http://pbs.twimg.com/media/FerF4bdVQAAKeYJ.jpg","media_url_https":"https://pbs.twimg.com/media/FerF4bdVQAAKeYJ.jpg","url":"https://t.co/VYKgE22wYb","display_url":"pic.twitter.com/VYKgE22wYb","expanded_url":"https://twitter.com/mexicanwilddog/status/1579292281898766336/video/1","type":"photo","original_info":{"width":2048,"height":1279,"focus_rects":[{"x":0,"y":0,"h":1147,"w":2048},{"x":640,"y":0,"h":1279,"w":1279},{"x":718,"y":0,"h":1279,"w":1122},{"x":959,"y":0,"h":1279,"w":640},{"x":0,"y":0,"h":1279,"w":2048}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":749,"resize":"fit"},"large":{"w":2048,"h":1279,"resize":"fit"},"small":{"w":680,"h":425,"resize":"fit"}},"media_key":"3_1579292192580911104","ext_media_availability":{"status":"available"},"ext_alt_text":null,"ext":{"mediaStats":{"r":"Missing","ttl":-1}}},{"id_str":"1579292197752430592","indices":[0,23],"media_url":"http://pbs.twimg.com/ext_tw_video_thumb/1579292197752430592/pu/img/soG4wMWOy3AVpllM.jpg","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1579292197752430592/pu/img/soG4wMWOy3AVpllM.jpg","url":"https://t.co/VYKgE22wYb","display_url":"pic.twitter.com/VYKgE22wYb","expanded_url":"https://twitter.com/mexicanwilddog/status/1579292281898766336/video/1","type":"video","original_info":{"width":640,"height":750},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":640,"h":750,"resize":"fit"},"small":{"w":580,"h":680,"resize":"fit"},"large":{"w":640,"h":750,"resize":"fit"}},"video_info":{"aspect_ratio":[64,75],"duration_millis":59929,"variants":[{"bitrate":2176000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1579292197752430592/pu/vid/640x750/UE-PSqG2EE5N2dN8.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1579292197752430592/pu/pl/X1U_gi6ol4Ah2QHl.m3u8?tag=12&container=fmp4"},{"bitrate":632000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1579292197752430592/pu/vid/320x374/5K6XeQ-v86lGPnPt.mp4?tag=12"},{"bitrate":950000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1579292197752430592/pu/vid/480x562/TmxxcK7S6dZQH4ZX.mp4?tag=12"}]},"media_key":"7_1579292197752430592","ext_media_availability":{"status":"available"},"ext_alt_text":null,"ext":{"mediaStats":{"r":{"ok":{"viewCount":"1175337"}},"ttl":-1}},"additional_media_info":{"monetizable":false}}]},"source":"Twitter for Android","user_id_str":"790285850","retweet_count":19727,"favorite_count":115546,"reply_count":366,"quote_count":951,"conversation_id_str":"1579292281898766336","possibly_sensitive_editable":true,"lang":"zxx","self_thread":{"id_str":"1579292281898766336"}} diff --git a/scraper/tweet.go b/scraper/tweet.go index a4dfcff..bf06d73 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -160,12 +160,18 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { if entity.Type != "video" && entity.Type != "animated_gif" { continue } - if len(apiTweet.ExtendedEntities.Media) != 1 { - panic(fmt.Errorf("Surprising ExtendedEntities: %v\n %w", apiTweet.ExtendedEntities.Media, EXTERNAL_API_ERROR)) + + new_video := ParseAPIVideo(entity, ret.ID) // This assigns TweetID + ret.Videos = append(ret.Videos, new_video) + + // Remove the thumbnail from the Images list + updated_imgs := []Image{} + for _, img := range ret.Images { + if VideoID(img.ID) != new_video.ID { + updated_imgs = append(updated_imgs, img) + } } - new_video := ParseAPIVideo(apiTweet.ExtendedEntities.Media[0], ret.ID) - ret.Videos = []Video{new_video} - ret.Images = []Image{} + ret.Images = updated_imgs } // Process polls diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index a54a3eb..3b31c4f 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -92,6 +92,36 @@ func TestParseTweetWithVideo(t *testing.T) { assert.False(v.IsGif) } +func TestParseTweetWith2Videos(t *testing.T) { + assert := assert.New(t) + tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_2_videos.json") + assert.Empty(tweet.Images) + assert.Len(tweet.Videos, 2) + + v1 := tweet.Videos[0] + assert.Equal("https://video.twimg.com/ext_tw_video/1579701730148847617/pu/vid/576x576/ghA0fyf58v-2naWR.mp4?tag=12", v1.RemoteURL) + assert.False(v1.IsGif) + v2 := tweet.Videos[1] + assert.Equal("https://video.twimg.com/ext_tw_video/1579701730157252608/pu/vid/480x480/VQ69Ut84XT2BgIzX.mp4?tag=12", v2.RemoteURL) + assert.False(v2.IsGif) +} + +func TestParseTweetWithImageAndVideo(t *testing.T) { + assert := assert.New(t) + tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_image_and_video.json") + assert.Len(tweet.Images, 1) + assert.Len(tweet.Videos, 1) + + img := tweet.Images[0] + assert.Equal(img.ID, ImageID(1579292192580911104)) + assert.Equal(img.RemoteURL, "https://pbs.twimg.com/media/FerF4bdVQAAKeYJ.jpg") + + vid := tweet.Videos[0] + assert.Equal(vid.ID, VideoID(1579292197752430592)) + assert.Equal(vid.ThumbnailRemoteUrl, "https://pbs.twimg.com/ext_tw_video_thumb/1579292197752430592/pu/img/soG4wMWOy3AVpllM.jpg") + assert.Equal(vid.RemoteURL, "https://video.twimg.com/ext_tw_video/1579292197752430592/pu/vid/640x750/UE-PSqG2EE5N2dN8.mp4?tag=12") +} + func TestParseTweetWithGif(t *testing.T) { assert := assert.New(t) tweet := load_tweet_from_file("test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json")