diff --git a/cmd/tests.sh b/cmd/tests.sh index 69aed60..4b4f93c 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -75,6 +75,22 @@ test $(sqlite3 twitter.db "select count(*) from tweets") = "2" test $(sqlite3 twitter.db "select count(*) from videos") = "1" +# Fetch a tweet with a GIF +tw fetch_user Cernovich +initial_videos_count=$(find videos | wc -l) +initial_videos_db_count=$(sqlite3 twitter.db "select count(*) from videos") +tw fetch_tweet_only https://twitter.com/Cernovich/status/1444429517020274693 + +test $(sqlite3 twitter.db "select count(*) from videos") = "$((initial_videos_db_count + 1))" +test $(sqlite3 twitter.db "select is_gif from videos where tweet_id = 1444429517020274693") = "1" + +# Download the GIF +test $(find videos | wc -l) = "$((initial_videos_count))" # Shouldn't have changed yet +tw download_tweet_content https://twitter.com/Cernovich/status/1444429517020274693 +test $(find videos | wc -l) = "$((initial_videos_count + 1))" + + + # Download a full thread tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1" diff --git a/persistence/media_queries.go b/persistence/media_queries.go index 8709417..06a55df 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error { */ func (p Profile) SaveVideo(vid scraper.Video) error { _, err := p.DB.Exec(` - insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded) - values (?, ?, ?, ?, ?) + insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif) + values (?, ?, ?, ?, ?, ?) on conflict do update set is_downloaded=? `, - vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, + vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif, vid.IsDownloaded, ) return err @@ -89,7 +89,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e * Get the list of videos for a tweet */ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) { - stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?") + stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded, is_gif from videos where tweet_id=?") if err != nil { return } @@ -100,7 +100,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e } var vid scraper.Video for rows.Next() { - err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded) + err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded, &vid.IsGif) if err != nil { return } diff --git a/persistence/media_queries_test.go b/persistence/media_queries_test.go index 440dcdd..32fd012 100644 --- a/persistence/media_queries_test.go +++ b/persistence/media_queries_test.go @@ -102,6 +102,7 @@ func TestSaveAndLoadVideo(t *testing.T) { rand.Seed(time.Now().UnixNano()) vid := create_video_from_id(rand.Int()) vid.TweetID = tweet.ID + vid.IsGif = true; // Save the Video err := profile.SaveVideo(vid) diff --git a/persistence/schema.sql b/persistence/schema.sql index d4b1976..67acd99 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -88,6 +88,7 @@ create table videos (rowid integer primary key, tweet_id integer not null, remote_url text not null unique, local_filename text not null unique, + is_gif boolean default 0, is_downloaded boolean default 0, foreign key(tweet_id) references tweets(id) diff --git a/persistence/utils_test.go b/persistence/utils_test.go index 070f0de..1aebf4c 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -89,6 +89,7 @@ func create_video_from_id(id int) scraper.Video { RemoteURL: filename, LocalFilename: filename, IsDownloaded: false, + IsGif: false, } } diff --git a/scraper/tweet.go b/scraper/tweet.go index 08cc57e..d202afc 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -129,7 +129,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID) for _, entity := range apiTweet.ExtendedEntities.Media { - if entity.Type != "video" { + if entity.Type != "video" && entity.Type != "animated_gif" { continue } if len(apiTweet.ExtendedEntities.Media) != 1 { diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 21e82c3..73bca1e 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -94,12 +94,30 @@ func TestParseTweetWithVideo(t *testing.T) { if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video { t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos) } + if tweet.Videos[0].IsGif != false { + t.Errorf("Expected it to be a regular video, but it was a gif") + } if len(tweet.Images) != 0 { t.Errorf("Should not have any images, but has %d", len(tweet.Images)) } } +func TestParseTweetWithGif(t *testing.T) { + tweet := load_tweet_from_file("test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json") + + expected_video := "https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4" + if len(tweet.Videos) != 1 { + t.Errorf("Expected 1 video (a gif), but got %d instead", len(tweet.Videos)) + } + if tweet.Videos[0].RemoteURL != expected_video { + t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos) + } + if tweet.Videos[0].IsGif != true { + t.Errorf("Expected video to be a gif, but it wasn't") + } +} + func TestParseTweetWithUrl(t *testing.T) { tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_url_card.json") diff --git a/scraper/video.go b/scraper/video.go index 2efc859..f198688 100644 --- a/scraper/video.go +++ b/scraper/video.go @@ -16,6 +16,7 @@ type Video struct { RemoteURL string LocalFilename string IsDownloaded bool + IsGif bool } func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video { @@ -30,5 +31,6 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video { RemoteURL: variants[0].URL, LocalFilename: local_filename, IsDownloaded: false, + IsGif: apiVideo.Type == "animated_gif", } }