diff --git a/cmd/tests.sh b/cmd/tests.sh index a2400f4..e57daef 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -20,5 +20,21 @@ test $(sqlite3 data/twitter.db "select count(*) from users") = "1" go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" test "$(sqlite3 data/twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do" + +# Try to double-download it go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" + + +# Fetch a tweet with a video +go run ./twitter fetch_user data DiamondChariots +test $(sqlite3 data/twitter.db "select handle from users" | wc -l) = "2" +go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 +test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" + +# Try to double-download it +go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 +test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" +test $(sqlite3 data/twitter.db "select count(*) from videos") = "1" + +echo -e "\033[32mAll tests passed. Finished successfully.\033[0m" diff --git a/persistence/media_download_test.go b/persistence/media_download_test.go deleted file mode 100644 index 3a0e73d..0000000 --- a/persistence/media_download_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package persistence_test - -import ( - "testing" - - "offline_twitter/scraper" -) - -/** - * Should return an `.mp4`file matching its parent Tweet's ID - */ -func TestVideoFilenameWhenDownloaded(t *testing.T) { - v := scraper.Video{TweetID: scraper.TweetID(23), IsDownloaded: false, Filename: "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/320x568/IXaQ5rPyf9mbD1aD.mp4?tag=12"} - outpath := v.FilenameWhenDownloaded() - expected := "23.mp4" - if outpath != expected { - t.Errorf("Expected output path to be %q, but got %q", expected, outpath) - } -} diff --git a/persistence/media_queries.go b/persistence/media_queries.go index 9fbd01a..9bbef93 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error { */ func (p Profile) SaveVideo(vid scraper.Video) error { _, err := p.DB.Exec(` - insert into videos (id, tweet_id, filename, is_downloaded) - values (?, ?, ?, ?) + insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded) + values (?, ?, ?, ?, ?) on conflict do update set is_downloaded=? `, - vid.ID, vid.TweetID, vid.Filename, vid.IsDownloaded, + vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsDownloaded, ) return err @@ -73,7 +73,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e * Get the list of videos for a tweet */ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) { - stmt, err := p.DB.Prepare("select id, filename, is_downloaded from videos where tweet_id=?") + stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?") if err != nil { return } @@ -84,7 +84,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e } var vid scraper.Video for rows.Next() { - err = rows.Scan(&vid.ID, &vid.Filename, &vid.IsDownloaded) + err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded) if err != nil { return } diff --git a/persistence/schema.sql b/persistence/schema.sql index f65f9bc..971fd2d 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -70,7 +70,8 @@ create table images (rowid integer primary key, create table videos (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), tweet_id integer not null, - filename text not null unique, + remote_url text not null unique, + local_filename text not null unique, is_downloaded boolean default 0, foreign key(tweet_id) references tweets(id) diff --git a/persistence/utils_test.go b/persistence/utils_test.go index aee6fa4..53b2a6b 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -79,7 +79,8 @@ func create_video_from_id(id int) scraper.Video { return scraper.Video{ ID: scraper.VideoID(id), TweetID: -1, - Filename: filename, + RemoteURL: filename, + LocalFilename: filename, IsDownloaded: false, } } diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index f7bdf5e..834d8fc 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -153,8 +153,8 @@ func TestParseTweetWithVideo(t *testing.T) { t.Errorf(err.Error()) } expected_video := "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12" - if len(tweet.Videos) != 1 || tweet.Videos[0].Filename != expected_video { - t.Errorf("Expected video %q, but got %+v", expected_video, tweet.Videos) + if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video { + t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos) } if len(tweet.Images) != 0 { diff --git a/scraper/video.go b/scraper/video.go index 725306f..2efc859 100644 --- a/scraper/video.go +++ b/scraper/video.go @@ -13,7 +13,6 @@ type VideoID int64 type Video struct { ID VideoID TweetID TweetID - Filename string // TODO video-filename: delete when it all works RemoteURL string LocalFilename string IsDownloaded bool @@ -28,13 +27,8 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video { return Video{ ID: VideoID(apiVideo.ID), TweetID: tweet_id, - Filename: variants[0].URL, RemoteURL: variants[0].URL, LocalFilename: local_filename, IsDownloaded: false, } } - -func (v Video) FilenameWhenDownloaded() string { // TODO video-filename: delete whole method and associated test - return fmt.Sprintf("%d.mp4", v.TweetID) -} diff --git a/scraper/video_test.go b/scraper/video_test.go index 9e8fe0d..f079971 100644 --- a/scraper/video_test.go +++ b/scraper/video_test.go @@ -32,9 +32,6 @@ func TestParseAPIVideo(t *testing.T) { if video.RemoteURL != expected_remote_url { t.Errorf("Expected %q, got %q", expected_remote_url, video.RemoteURL) } - if video.Filename != expected_remote_url { // TODO video-filename: delete this check - t.Errorf("Expected %q, got %q", expected_remote_url, video.Filename) - } expected_local_filename := "28.mp4" if video.LocalFilename != expected_local_filename {