Fix gifs

cmd/tests.sh

@@ -75,6 +75,22 @@ test $(sqlite3 twitter.db "select count(*) from tweets") = "2"
 test $(sqlite3 twitter.db "select count(*) from videos") = "1"
 
 
+# Fetch a tweet with a GIF
+tw fetch_user Cernovich
+initial_videos_count=$(find videos | wc -l)
+initial_videos_db_count=$(sqlite3 twitter.db "select count(*) from videos")
+tw fetch_tweet_only https://twitter.com/Cernovich/status/1444429517020274693
+
+test $(sqlite3 twitter.db "select count(*) from videos") = "$((initial_videos_db_count + 1))"
+test $(sqlite3 twitter.db "select is_gif from videos where tweet_id = 1444429517020274693") = "1"
+
+# Download the GIF
+test $(find videos | wc -l) = "$((initial_videos_count))"   # Shouldn't have changed yet
+tw download_tweet_content https://twitter.com/Cernovich/status/1444429517020274693
+test $(find videos | wc -l) = "$((initial_videos_count + 1))"
+
+
+
 # Download a full thread
 tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867
 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1"

persistence/media_queries.go

@@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error {
  */
 func (p Profile) SaveVideo(vid scraper.Video) error {
     _, err := p.DB.Exec(`
-        insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded)
+        insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif)
-                    values (?, ?, ?, ?, ?)
+                    values (?, ?, ?, ?, ?, ?)
                on conflict do update
                        set is_downloaded=?
         `,
-        vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded,
+        vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif,
         vid.IsDownloaded,
     )
     return err
@@ -89,7 +89,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
  * Get the list of videos for a tweet
  */
 func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
-    stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?")
+    stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded, is_gif from videos where tweet_id=?")
     if err != nil {
         return
     }
@@ -100,7 +100,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e
     }
     var vid scraper.Video
     for rows.Next() {
-        err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded)
+        err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded, &vid.IsGif)
         if err != nil {
             return
         }

persistence/media_queries_test.go

@@ -102,6 +102,7 @@ func TestSaveAndLoadVideo(t *testing.T) {
     rand.Seed(time.Now().UnixNano())
     vid := create_video_from_id(rand.Int())
     vid.TweetID = tweet.ID
+    vid.IsGif = true;
 
     // Save the Video
     err := profile.SaveVideo(vid)

persistence/schema.sql

@@ -88,6 +88,7 @@ create table videos (rowid integer primary key,
     tweet_id integer not null,
     remote_url text not null unique,
     local_filename text not null unique,
+    is_gif boolean default 0,
     is_downloaded boolean default 0,
 
     foreign key(tweet_id) references tweets(id)

persistence/utils_test.go

@@ -89,6 +89,7 @@ func create_video_from_id(id int) scraper.Video {
 		RemoteURL: filename,
 		LocalFilename: filename,
 		IsDownloaded: false,
+		IsGif: false,
 	}
 }
 

scraper/tweet.go

@@ -129,7 +129,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
 	ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID)
 
 	for _, entity := range apiTweet.ExtendedEntities.Media {
-		if entity.Type != "video" {
+		if entity.Type != "video" && entity.Type != "animated_gif" {
 			continue
 		}
 		if len(apiTweet.ExtendedEntities.Media) != 1 {

scraper/tweet_test.go

@@ -94,12 +94,30 @@ func TestParseTweetWithVideo(t *testing.T) {
 	if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video {
 		t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
 	}
+	if tweet.Videos[0].IsGif != false {
+		t.Errorf("Expected it to be a regular video, but it was a gif")
+	}
 
 	if len(tweet.Images) != 0 {
 		t.Errorf("Should not have any images, but has %d", len(tweet.Images))
 	}
 }
 
+func TestParseTweetWithGif(t *testing.T) {
+	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json")
+
+	expected_video := "https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4"
+	if len(tweet.Videos) != 1 {
+		t.Errorf("Expected 1 video (a gif), but got %d instead", len(tweet.Videos))
+	}
+	if tweet.Videos[0].RemoteURL != expected_video {
+		t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
+	}
+	if tweet.Videos[0].IsGif != true {
+		t.Errorf("Expected video to be a gif, but it wasn't")
+	}
+}
+
 func TestParseTweetWithUrl(t *testing.T) {
 	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_url_card.json")
 

scraper/video.go

@@ -16,6 +16,7 @@ type Video struct {
     RemoteURL string
     LocalFilename string
     IsDownloaded bool
+    IsGif  bool
 }
 
 func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
@@ -30,5 +31,6 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
         RemoteURL: variants[0].URL,
         LocalFilename: local_filename,
         IsDownloaded: false,
+        IsGif: apiVideo.Type == "animated_gif",
     }
 }