From 8462365d70a5e7dae1b309edcf9cba5111662ac6 Mon Sep 17 00:00:00 2001 From: Alessio Date: Wed, 4 Aug 2021 23:49:45 -0700 Subject: [PATCH] Add tests for API video parsing --- scraper/test_responses/video.json | 1 + scraper/video_test.go | 46 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 scraper/test_responses/video.json create mode 100644 scraper/video_test.go diff --git a/scraper/test_responses/video.json b/scraper/test_responses/video.json new file mode 100644 index 0000000..e4d10fd --- /dev/null +++ b/scraper/test_responses/video.json @@ -0,0 +1 @@ +{"id_str":"1418951950020845568","indices":[10,33],"media_url":"http://pbs.twimg.com/ext_tw_video_thumb/1418951950020845568/pu/img/eUTaYYfuAJ8FyjUi.jpg","media_url_https":"https://pbs.twimg.com/ext_tw_video_thumb/1418951950020845568/pu/img/eUTaYYfuAJ8FyjUi.jpg","url":"https://t.co/HfIexbaCLe","display_url":"pic.twitter.com/HfIexbaCLe","expanded_url":"https://twitter.com/crazykarens/status/1418953196484038665/video/1","type":"video","original_info":{"width":720,"height":1280},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":383,"h":680,"resize":"fit"},"large":{"w":720,"h":1280,"resize":"fit"},"medium":{"w":675,"h":1200,"resize":"fit"}},"source_status_id_str":"1418953196484038665","source_user_id_str":"1269387380127080448","video_info":{"aspect_ratio":[9,16],"duration_millis":88300,"variants":[{"bitrate":632000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/320x568/IXaQ5rPyf9mbD1aD.mp4?tag=12"},{"bitrate":950000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/480x852/Z3vRCGq3hctSx7qv.mp4?tag=12"},{"content_type":"application/x-mpegURL","url":"https://video.twimg.com/ext_tw_video/1418951950020845568/pu/pl/cB33qJYlO9sdI44P.m3u8?tag=12&container=fmp4"},{"bitrate":2176000,"content_type":"video/mp4","url":"https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12"}]},"media_key":"7_1418951950020845568","ext_alt_text":null,"ext_media_availability":{"status":"available"},"ext_media_color":{"palette":[{"rgb":{"red":57,"green":59,"blue":64},"percentage":48.69},{"rgb":{"red":123,"green":137,"blue":108},"percentage":19.14},{"rgb":{"red":109,"green":112,"blue":133},"percentage":10.69},{"rgb":{"red":135,"green":158,"blue":98},"percentage":4.7},{"rgb":{"red":227,"green":214,"blue":206},"percentage":2.42}]},"ext":{"mediaStats":{"r":{"ok":{"viewCount":"275952"}},"ttl":-1}},"additional_media_info":{"monetizable":false,"source_user":{"id_str":"1269387380127080448","name":"Karen","screen_name":"crazykarens","location":"Karen","description":"EXCLUSIVE Karen Content & Updates","entities":{"description":{}},"followers_count":57393,"fast_followers_count":0,"normal_followers_count":57393,"friends_count":0,"listed_count":122,"created_at":"Sat Jun 06 21:55:18 +0000 2020","favourites_count":5,"statuses_count":566,"media_count":489,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1398068069830901761/6R8k799k_normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1269387380127080448/1591619036","profile_image_extensions_alt_text":null,"profile_image_extensions_media_availability":null,"profile_image_extensions_media_color":{"palette":[{"rgb":{"red":255,"green":255,"blue":255},"percentage":84.59},{"rgb":{"red":43,"green":43,"blue":43},"percentage":9.22}]},"profile_image_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_banner_extensions_media_color":{"palette":[{"rgb":{"red":247,"green":247,"blue":247},"percentage":90.06},{"rgb":{"red":33,"green":33,"blue":33},"percentage":6.29},{"rgb":{"red":119,"green":119,"blue":119},"percentage":4.94}]},"profile_banner_extensions_alt_text":null,"profile_banner_extensions_media_availability":null,"profile_banner_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_link_color":"1DA1F2","has_extended_profile":true,"default_profile":true,"pinned_tweet_ids":[],"pinned_tweet_ids_str":[],"has_custom_timelines":true,"advertiser_account_type":"none","advertiser_account_service_levels":[],"profile_interstitial_type":"","business_profile_state":"none","translator_type":"none","withheld_in_countries":[],"ext":{"highlightedLabel":{"r":{"ok":{}},"ttl":-1}}}}} diff --git a/scraper/video_test.go b/scraper/video_test.go new file mode 100644 index 0000000..9e8fe0d --- /dev/null +++ b/scraper/video_test.go @@ -0,0 +1,46 @@ +package scraper_test + +import ( + "testing" + "io/ioutil" + "encoding/json" + + "offline_twitter/scraper" +) + +func TestParseAPIVideo(t *testing.T) { + data, err := ioutil.ReadFile("test_responses/video.json") + if err != nil { + panic(err) + } + var apivideo scraper.APIExtendedMedia + err = json.Unmarshal(data, &apivideo) + if err != nil { + t.Fatal(err.Error()) + } + tweet_id := scraper.TweetID(28) + video := scraper.ParseAPIVideo(apivideo, tweet_id) + + expected_id := 1418951950020845568 + if video.ID != scraper.VideoID(expected_id) { + t.Errorf("Expected ID of %d, got %d", expected_id, video.ID) + } + if video.TweetID != tweet_id { + t.Errorf("Expected ID of %d, got %d", tweet_id, video.TweetID) + } + expected_remote_url := "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12" + if video.RemoteURL != expected_remote_url { + t.Errorf("Expected %q, got %q", expected_remote_url, video.RemoteURL) + } + if video.Filename != expected_remote_url { // TODO video-filename: delete this check + t.Errorf("Expected %q, got %q", expected_remote_url, video.Filename) + } + + expected_local_filename := "28.mp4" + if video.LocalFilename != expected_local_filename { + t.Errorf("Expected %q, got %q", expected_local_filename, video.LocalFilename) + } + if video.IsDownloaded { + t.Errorf("Expected it not to be downloaded, but it was") + } +}