diff --git a/persistence/media_queries.go b/persistence/media_queries.go index 85e665b..3eb7867 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -7,22 +7,22 @@ import ( ) /** - * Save an Image. If it's a new Image (no rowid), does an insert; otherwise, does an update. + * Save an Image * * args: * - img: the Image to save - * - * returns: - * - the rowid */ -func (p Profile) SaveImage(img scraper.Image) (sql.Result, error) { - if img.ID == 0 { - // New image - return p.DB.Exec("insert into images (tweet_id, filename) values (?, ?) on conflict do nothing", img.TweetID, img.Filename) - } else { - // Updating an existing image - return p.DB.Exec("update images set filename=?, is_downloaded=? where rowid=?", img.Filename, img.IsDownloaded, img.ID) - } +func (p Profile) SaveImage(img scraper.Image) error { + _, err := p.DB.Exec(` + insert into images (id, tweet_id, filename, is_downloaded) + values (?, ?, ?, ?, ?, ?) + on conflict do update + set is_downloaded=? + `, + img.ID, img.TweetID, img.Filename, img.IsDownloaded, + img.IsDownloaded, + ) + return err } /** @@ -48,7 +48,7 @@ func (p Profile) SaveVideo(vid scraper.Video) (sql.Result, error) { * Get the list of images for a tweet */ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) { - stmt, err := p.DB.Prepare("select rowid, filename, is_downloaded from images where tweet_id=?") + stmt, err := p.DB.Prepare("select id, filename, is_downloaded from images where tweet_id=?") if err != nil { return } diff --git a/persistence/media_queries_test.go b/persistence/media_queries_test.go index fdf19f3..8d3a7cc 100644 --- a/persistence/media_queries_test.go +++ b/persistence/media_queries_test.go @@ -23,19 +23,14 @@ func TestSaveAndLoadImage(t *testing.T) { // Create a fresh Image to test on rand.Seed(time.Now().UnixNano()) - filename := fmt.Sprint(rand.Int()) - img := scraper.Image{TweetID: tweet.ID, Filename: filename, IsDownloaded: false} + img := create_image_from_id(rand.Int()) + img.TweetID = tweet.ID // Save the Image - result, err := profile.SaveImage(img) + err := profile.SaveImage(img) if err != nil { t.Fatalf("Failed to save the image: %s", err.Error()) } - last_insert, err := result.LastInsertId() - if err != nil { - t.Fatalf("last insert??? %s", err.Error()) - } - img.ID = scraper.ImageID(last_insert) // Reload the Image imgs, err := profile.GetImagesForTweet(tweet) @@ -67,25 +62,17 @@ func TestModifyImage(t *testing.T) { tweet := create_stable_tweet() img := tweet.Images[0] - if img.ID != 1 { - t.Fatalf("Got the wrong image back: wanted ID %d, got %d", 1, img.ID) + if img.ID != -1 { + t.Fatalf("Got the wrong image back: wanted ID %d, got %d", -1, img.ID) } - img.Filename = "local/sdfjk.jpg" img.IsDownloaded = true // Save the changes - result, err := profile.SaveImage(img) + err := profile.SaveImage(img) if err != nil { t.Error(err) } - rows_affected, err := result.RowsAffected() - if err != nil { - t.Error(err) - } - if rows_affected != 1 { - t.Errorf("Expected 1 row changed, but got %d", rows_affected) - } // Reload it imgs, err := profile.GetImagesForTweet(tweet) @@ -94,7 +81,7 @@ func TestModifyImage(t *testing.T) { } new_img := imgs[0] if new_img.ID != img.ID { - t.Fatalf("Got the wrong image back: wanted ID %d, got %d", 1, new_img.ID) + t.Fatalf("Got the wrong image back: wanted ID %d, got %d", -1, new_img.ID) } if diff := deep.Equal(img, new_img); diff != nil { diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go index fb34323..9539f45 100644 --- a/persistence/tweet_queries.go +++ b/persistence/tweet_queries.go @@ -39,7 +39,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { } } for _, image := range t.Images { - _, err := p.SaveImage(image) + err := p.SaveImage(image) if err != nil { return err } diff --git a/persistence/tweet_queries_test.go b/persistence/tweet_queries_test.go index cfa8efa..0eadcaa 100644 --- a/persistence/tweet_queries_test.go +++ b/persistence/tweet_queries_test.go @@ -28,11 +28,6 @@ func TestSaveAndLoadTweet(t *testing.T) { t.Fatalf("Failed to load the tweet: %s", err.Error()) } - // Spoof the image and video IDs - // TODO: This feels clumsy-- possible bad design - for i := range tweet.Images { - tweet.Images[i].ID = new_tweet.Images[i].ID - } for i := range tweet.Videos { tweet.Videos[i].ID = new_tweet.Videos[i].ID } diff --git a/persistence/utils_test.go b/persistence/utils_test.go index eda1238..23d0175 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -58,6 +58,18 @@ func create_stable_user() scraper.User { } } +/** + * Create a semi-stable image based on the given ID + */ +func create_image_from_id(id int) scraper.Image { + filename := fmt.Sprintf("image%d.jpg", id) + return scraper.Image{ + ID: scraper.ImageID(id), + TweetID: "-1", + Filename: filename, + IsDownloaded: false, + } +} /** * Create a stable tweet with a fixed ID and content @@ -75,7 +87,9 @@ func create_stable_tweet() scraper.Tweet { NumQuoteTweets: 10, Videos: []scraper.Video{{ID: scraper.VideoID(1), TweetID: tweet_id, Filename: "asdf", IsDownloaded: false}}, Urls: []string{}, - Images: []scraper.Image{{ID: scraper.ImageID(1), TweetID: tweet_id, Filename: "asdf", IsDownloaded: false}}, + Images: []scraper.Image{ + create_image_from_id(-1), + }, Mentions: []scraper.UserHandle{}, Hashtags: []string{}, } @@ -115,6 +129,11 @@ func create_dummy_tweet() scraper.Tweet { rand.Seed(time.Now().UnixNano()) tweet_id := scraper.TweetID(fmt.Sprint(rand.Int())) + img1 := create_image_from_id(rand.Int()) + img1.TweetID = tweet_id + img2 := create_image_from_id(rand.Int()) + img2.TweetID = tweet_id + return scraper.Tweet{ ID: tweet_id, UserID: scraper.UserID("-1"), @@ -126,10 +145,7 @@ func create_dummy_tweet() scraper.Tweet { NumQuoteTweets: 4, Videos: []scraper.Video{scraper.Video{TweetID: tweet_id, Filename: "video" + string(tweet_id), IsDownloaded: false}}, Urls: []string{"url1", "url2"}, - Images: []scraper.Image{ - scraper.Image{TweetID: tweet_id, Filename: "image1" + string(tweet_id), IsDownloaded: false}, - scraper.Image{TweetID: tweet_id, Filename: "image2" + string(tweet_id), IsDownloaded: false}, - }, + Images: []scraper.Image{img1, img2}, Mentions: []scraper.UserHandle{"mention1", "mention2"}, Hashtags: []string{"hash1", "hash2"}, } diff --git a/scraper/api_types.go b/scraper/api_types.go index 47d3b78..cf906a0 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -15,6 +15,7 @@ func (v SortableVariants) Swap(i, j int) { v[i], v[j] = v[j], v[i] } func (v SortableVariants) Less(i, j int) bool { return v[i].Bitrate > v[j].Bitrate } type APIMedia struct { + ID int64 `json:"id_str,string"` MediaURLHttps string `json:"media_url_https"` Type string `json:"type"` URL string `json:"url"` diff --git a/scraper/image.go b/scraper/image.go index 1a104f6..9b28e31 100644 --- a/scraper/image.go +++ b/scraper/image.go @@ -4,7 +4,7 @@ import ( "path" ) -type ImageID int +type ImageID int64 type Image struct { ID ImageID @@ -18,6 +18,7 @@ type Image struct { func ParseAPIMedia(apiMedia APIMedia) Image { local_filename := path.Base(apiMedia.MediaURLHttps) return Image{ + ID: ImageID(apiMedia.ID), Filename: apiMedia.MediaURLHttps, // XXX filename RemoteURL: apiMedia.MediaURLHttps, LocalFilename: local_filename, diff --git a/scraper/image_test.go b/scraper/image_test.go index 00deec3..4216874 100644 --- a/scraper/image_test.go +++ b/scraper/image_test.go @@ -20,6 +20,10 @@ func TestParseAPIMedia(t *testing.T) { } image := scraper.ParseAPIMedia(apimedia) + expected_id := 1395882862289772553 + if image.ID != scraper.ImageID(expected_id) { + t.Errorf("Expected ID of %q, got %q", expected_id, image.ID) + } expected_remote_url := "https://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg" if image.RemoteURL != expected_remote_url { t.Errorf("Expected %q, got %q", expected_remote_url, image.RemoteURL)