From d1ba8b48f39e619cd32e6415abce58394c3b83f7 Mon Sep 17 00:00:00 2001 From: Alessio Date: Thu, 5 Aug 2021 14:22:16 -0700 Subject: [PATCH] Add `RemoteURL` and `LocalFilename` to Images, remove `Filename` --- persistence/media_queries.go | 10 +++++----- persistence/schema.sql | 3 ++- persistence/utils_test.go | 3 ++- scraper/image.go | 6 ------ scraper/image_test.go | 3 --- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/persistence/media_queries.go b/persistence/media_queries.go index 9bbef93..f997406 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -12,12 +12,12 @@ import ( */ func (p Profile) SaveImage(img scraper.Image) error { _, err := p.DB.Exec(` - insert into images (id, tweet_id, filename, is_downloaded) - values (?, ?, ?, ?) + insert into images (id, tweet_id, remote_url, local_filename, is_downloaded) + values (?, ?, ?, ?, ?) on conflict do update set is_downloaded=? `, - img.ID, img.TweetID, img.Filename, img.IsDownloaded, + img.ID, img.TweetID, img.RemoteURL, img.LocalFilename, img.IsDownloaded, img.IsDownloaded, ) return err @@ -46,7 +46,7 @@ func (p Profile) SaveVideo(vid scraper.Video) error { * Get the list of images for a tweet */ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) { - stmt, err := p.DB.Prepare("select id, filename, is_downloaded from images where tweet_id=?") + stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from images where tweet_id=?") if err != nil { return } @@ -58,7 +58,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e var img scraper.Image for rows.Next() { - err = rows.Scan(&img.ID, &img.Filename, &img.IsDownloaded) + err = rows.Scan(&img.ID, &img.RemoteURL, &img.LocalFilename, &img.IsDownloaded) if err != nil { return } diff --git a/persistence/schema.sql b/persistence/schema.sql index 971fd2d..ed1cece 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -61,7 +61,8 @@ create table urls (rowid integer primary key, create table images (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), tweet_id integer not null, - filename text not null unique, + remote_url text not null unique, + local_filename text not null unique, is_downloaded boolean default 0, foreign key(tweet_id) references tweets(id) diff --git a/persistence/utils_test.go b/persistence/utils_test.go index 53b2a6b..427ec81 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -66,7 +66,8 @@ func create_image_from_id(id int) scraper.Image { return scraper.Image{ ID: scraper.ImageID(id), TweetID: -1, - Filename: filename, + RemoteURL: filename, + LocalFilename: filename, IsDownloaded: false, } } diff --git a/scraper/image.go b/scraper/image.go index 55c8426..4b67ac8 100644 --- a/scraper/image.go +++ b/scraper/image.go @@ -9,7 +9,6 @@ type ImageID int64 type Image struct { ID ImageID TweetID TweetID - Filename string RemoteURL string LocalFilename string IsDownloaded bool @@ -19,13 +18,8 @@ func ParseAPIMedia(apiMedia APIMedia) Image { local_filename := path.Base(apiMedia.MediaURLHttps) return Image{ ID: ImageID(apiMedia.ID), - Filename: apiMedia.MediaURLHttps, // TODO filename RemoteURL: apiMedia.MediaURLHttps, LocalFilename: local_filename, IsDownloaded: false, } } - -func (img Image) FilenameWhenDownloaded() string { - return path.Base(img.Filename) -} diff --git a/scraper/image_test.go b/scraper/image_test.go index 4216874..2821697 100644 --- a/scraper/image_test.go +++ b/scraper/image_test.go @@ -28,9 +28,6 @@ func TestParseAPIMedia(t *testing.T) { if image.RemoteURL != expected_remote_url { t.Errorf("Expected %q, got %q", expected_remote_url, image.RemoteURL) } - if image.Filename != expected_remote_url { // XXX filename: delete this check - t.Errorf("Expected %q, got %q", expected_remote_url, image.Filename) - } expected_local_filename := "E18sEUrWYAk8dBl.jpg" if image.LocalFilename != expected_local_filename { t.Errorf("Expected %q, got %q", expected_local_filename, image.LocalFilename)