Add RemoteURL and LocalFilename fields to Video, remove Filename

cmd/tests.sh

@@ -20,5 +20,21 @@ test $(sqlite3 data/twitter.db "select count(*) from users") = "1"
 go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700
 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1"
 test "$(sqlite3 data/twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do"
+
+# Try to double-download it
 go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700
 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1"
+
+
+# Fetch a tweet with a video
+go run ./twitter fetch_user data DiamondChariots
+test $(sqlite3 data/twitter.db "select handle from users" | wc -l) = "2"
+go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340
+test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2"
+
+# Try to double-download it
+go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340
+test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2"
+test $(sqlite3 data/twitter.db "select count(*) from videos") = "1"
+
+echo -e "\033[32mAll tests passed.  Finished successfully.\033[0m"

persistence/media_download_test.go

@@ -1,19 +0,0 @@
-package persistence_test
-
-import (
-    "testing"
-
-    "offline_twitter/scraper"
-)
-
-/**
- * Should return an `.mp4`file matching its parent Tweet's ID
- */
-func TestVideoFilenameWhenDownloaded(t *testing.T) {
-    v := scraper.Video{TweetID: scraper.TweetID(23), IsDownloaded: false, Filename: "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/320x568/IXaQ5rPyf9mbD1aD.mp4?tag=12"}
-    outpath := v.FilenameWhenDownloaded()
-    expected := "23.mp4"
-    if outpath != expected {
-        t.Errorf("Expected output path to be %q, but got %q", expected, outpath)
-    }
-}

persistence/media_queries.go

@@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error {
  */
 func (p Profile) SaveVideo(vid scraper.Video) error {
     _, err := p.DB.Exec(`
-        insert into videos (id, tweet_id, filename, is_downloaded)
+        insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded)
-                    values (?, ?, ?, ?)
+                    values (?, ?, ?, ?, ?)
                on conflict do update
                        set is_downloaded=?
         `,
-        vid.ID, vid.TweetID, vid.Filename, vid.IsDownloaded,
+        vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded,
         vid.IsDownloaded,
     )
     return err
@@ -73,7 +73,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
  * Get the list of videos for a tweet
  */
 func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
-    stmt, err := p.DB.Prepare("select id, filename, is_downloaded from videos where tweet_id=?")
+    stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?")
     if err != nil {
         return
     }
@@ -84,7 +84,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e
     }
     var vid scraper.Video
     for rows.Next() {
-        err = rows.Scan(&vid.ID, &vid.Filename, &vid.IsDownloaded)
+        err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded)
         if err != nil {
             return
         }

persistence/schema.sql

@@ -70,7 +70,8 @@ create table images (rowid integer primary key,
 create table videos (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
     tweet_id integer not null,
-    filename text not null unique,
+    remote_url text not null unique,
+    local_filename text not null unique,
     is_downloaded boolean default 0,
 
     foreign key(tweet_id) references tweets(id)

persistence/utils_test.go

@@ -79,7 +79,8 @@ func create_video_from_id(id int) scraper.Video {
 	return scraper.Video{
 		ID: scraper.VideoID(id),
 		TweetID: -1,
-		Filename: filename,
+		RemoteURL: filename,
+		LocalFilename: filename,
 		IsDownloaded: false,
 	}
 }

scraper/tweet_test.go

@@ -153,8 +153,8 @@ func TestParseTweetWithVideo(t *testing.T) {
 		t.Errorf(err.Error())
 	}
 	expected_video := "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12"
-	if len(tweet.Videos) != 1 || tweet.Videos[0].Filename != expected_video {
+	if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video {
-		t.Errorf("Expected video %q, but got %+v", expected_video, tweet.Videos)
+		t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
 	}
 
 	if len(tweet.Images) != 0 {

scraper/video.go

@@ -13,7 +13,6 @@ type VideoID int64
 type Video struct {
     ID VideoID
     TweetID TweetID
-    Filename string  // TODO video-filename: delete when it all works
     RemoteURL string
     LocalFilename string
     IsDownloaded bool
@@ -28,13 +27,8 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
     return Video{
         ID: VideoID(apiVideo.ID),
         TweetID: tweet_id,
-        Filename: variants[0].URL,
         RemoteURL: variants[0].URL,
         LocalFilename: local_filename,
         IsDownloaded: false,
     }
 }
-
-func (v Video) FilenameWhenDownloaded() string {  // TODO video-filename: delete whole method and associated test
-    return fmt.Sprintf("%d.mp4", v.TweetID)
-}

scraper/video_test.go

@@ -32,9 +32,6 @@ func TestParseAPIVideo(t *testing.T) {
     if video.RemoteURL != expected_remote_url {
         t.Errorf("Expected %q, got %q", expected_remote_url, video.RemoteURL)
     }
-    if video.Filename != expected_remote_url { // TODO video-filename: delete this check
-        t.Errorf("Expected %q, got %q", expected_remote_url, video.Filename)
-    }
 
     expected_local_filename := "28.mp4"
     if video.LocalFilename != expected_local_filename {