me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use ImageID from the API rather than rowid

Browse Source
This commit is contained in:
Alessio 2021-08-04 01:23:55 -07:00
parent 494ca25dc4
commit 7a81b71cd3
8 changed files with 49 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
persistence/media_queries.go
View File

@ -7,22 +7,22 @@ import (
)
/**
* Save an Image. If it's a new Image (no rowid), does an insert; otherwise, does an update.
* Save an Image
*
* args:
* - img: the Image to save
*
* returns:
* - the rowid
*/
func (p Profile) SaveImage(img scraper.Image) (sql.Result, error) {
if img.ID == 0 {
// New image
return p.DB.Exec("insert into images (tweet_id, filename) values (?, ?) on conflict do nothing", img.TweetID, img.Filename)
} else {
// Updating an existing image
return p.DB.Exec("update images set filename=?, is_downloaded=? where rowid=?", img.Filename, img.IsDownloaded, img.ID)
}
func (p Profile) SaveImage(img scraper.Image) error {
_, err := p.DB.Exec(`
insert into images (id, tweet_id, filename, is_downloaded)
values (?, ?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=?
`,
img.ID, img.TweetID, img.Filename, img.IsDownloaded,
img.IsDownloaded,
)
return err
}
/**
@ -48,7 +48,7 @@ func (p Profile) SaveVideo(vid scraper.Video) (sql.Result, error) {
* Get the list of images for a tweet
*/
func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) {
stmt, err := p.DB.Prepare("select rowid, filename, is_downloaded from images where tweet_id=?")
stmt, err := p.DB.Prepare("select id, filename, is_downloaded from images where tweet_id=?")
if err != nil {
return
}

27
persistence/media_queries_test.go
View File

@ -23,19 +23,14 @@ func TestSaveAndLoadImage(t *testing.T) {
// Create a fresh Image to test on
rand.Seed(time.Now().UnixNano())
filename := fmt.Sprint(rand.Int())
img := scraper.Image{TweetID: tweet.ID, Filename: filename, IsDownloaded: false}
img := create_image_from_id(rand.Int())
img.TweetID = tweet.ID
// Save the Image
result, err := profile.SaveImage(img)
err := profile.SaveImage(img)
if err != nil {
t.Fatalf("Failed to save the image: %s", err.Error())
}
last_insert, err := result.LastInsertId()
if err != nil {
t.Fatalf("last insert??? %s", err.Error())
}
img.ID = scraper.ImageID(last_insert)
// Reload the Image
imgs, err := profile.GetImagesForTweet(tweet)
@ -67,25 +62,17 @@ func TestModifyImage(t *testing.T) {
tweet := create_stable_tweet()
img := tweet.Images[0]
if img.ID != 1 {
t.Fatalf("Got the wrong image back: wanted ID %d, got %d", 1, img.ID)
if img.ID != -1 {
t.Fatalf("Got the wrong image back: wanted ID %d, got %d", -1, img.ID)
}
img.Filename = "local/sdfjk.jpg"
img.IsDownloaded = true
// Save the changes
result, err := profile.SaveImage(img)
err := profile.SaveImage(img)
if err != nil {
t.Error(err)
}
rows_affected, err := result.RowsAffected()
if err != nil {
t.Error(err)
}
if rows_affected != 1 {
t.Errorf("Expected 1 row changed, but got %d", rows_affected)
}
// Reload it
imgs, err := profile.GetImagesForTweet(tweet)
@ -94,7 +81,7 @@ func TestModifyImage(t *testing.T) {
}
new_img := imgs[0]
if new_img.ID != img.ID {
t.Fatalf("Got the wrong image back: wanted ID %d, got %d", 1, new_img.ID)
t.Fatalf("Got the wrong image back: wanted ID %d, got %d", -1, new_img.ID)
}
if diff := deep.Equal(img, new_img); diff != nil {

2
persistence/tweet_queries.go
View File

@ -39,7 +39,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
}
}
for _, image := range t.Images {
_, err := p.SaveImage(image)
err := p.SaveImage(image)
if err != nil {
return err
}

5
persistence/tweet_queries_test.go
View File

@ -28,11 +28,6 @@ func TestSaveAndLoadTweet(t *testing.T) {
t.Fatalf("Failed to load the tweet: %s", err.Error())
}
// Spoof the image and video IDs
// TODO: This feels clumsy-- possible bad design
for i := range tweet.Images {
tweet.Images[i].ID = new_tweet.Images[i].ID
}
for i := range tweet.Videos {
tweet.Videos[i].ID = new_tweet.Videos[i].ID
}

26
persistence/utils_test.go
View File

@ -58,6 +58,18 @@ func create_stable_user() scraper.User {
}
}
/**
* Create a semi-stable image based on the given ID
*/
func create_image_from_id(id int) scraper.Image {
filename := fmt.Sprintf("image%d.jpg", id)
return scraper.Image{
ID: scraper.ImageID(id),
TweetID: "-1",
Filename: filename,
IsDownloaded: false,
}
}
/**
* Create a stable tweet with a fixed ID and content
@ -75,7 +87,9 @@ func create_stable_tweet() scraper.Tweet {
NumQuoteTweets: 10,
Videos: []scraper.Video{{ID: scraper.VideoID(1), TweetID: tweet_id, Filename: "asdf", IsDownloaded: false}},
Urls: []string{},
Images: []scraper.Image{{ID: scraper.ImageID(1), TweetID: tweet_id, Filename: "asdf", IsDownloaded: false}},
Images: []scraper.Image{
create_image_from_id(-1),
},
Mentions: []scraper.UserHandle{},
Hashtags: []string{},
}
@ -115,6 +129,11 @@ func create_dummy_tweet() scraper.Tweet {
rand.Seed(time.Now().UnixNano())
tweet_id := scraper.TweetID(fmt.Sprint(rand.Int()))
img1 := create_image_from_id(rand.Int())
img1.TweetID = tweet_id
img2 := create_image_from_id(rand.Int())
img2.TweetID = tweet_id
return scraper.Tweet{
ID: tweet_id,
UserID: scraper.UserID("-1"),
@ -126,10 +145,7 @@ func create_dummy_tweet() scraper.Tweet {
NumQuoteTweets: 4,
Videos: []scraper.Video{scraper.Video{TweetID: tweet_id, Filename: "video" + string(tweet_id), IsDownloaded: false}},
Urls: []string{"url1", "url2"},
Images: []scraper.Image{
scraper.Image{TweetID: tweet_id, Filename: "image1" + string(tweet_id), IsDownloaded: false},
scraper.Image{TweetID: tweet_id, Filename: "image2" + string(tweet_id), IsDownloaded: false},
},
Images: []scraper.Image{img1, img2},
Mentions: []scraper.UserHandle{"mention1", "mention2"},
Hashtags: []string{"hash1", "hash2"},
}

1
scraper/api_types.go
View File

@ -15,6 +15,7 @@ func (v SortableVariants) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v SortableVariants) Less(i, j int) bool { return v[i].Bitrate > v[j].Bitrate }
type APIMedia struct {
ID int64 `json:"id_str,string"`
MediaURLHttps string `json:"media_url_https"`
Type string `json:"type"`
URL string `json:"url"`

3
scraper/image.go
View File

@ -4,7 +4,7 @@ import (
"path"
)
type ImageID int
type ImageID int64
type Image struct {
ID ImageID
@ -18,6 +18,7 @@ type Image struct {
func ParseAPIMedia(apiMedia APIMedia) Image {
local_filename := path.Base(apiMedia.MediaURLHttps)
return Image{
ID: ImageID(apiMedia.ID),
Filename: apiMedia.MediaURLHttps, // XXX filename
RemoteURL: apiMedia.MediaURLHttps,
LocalFilename: local_filename,

4
scraper/image_test.go
View File

@ -20,6 +20,10 @@ func TestParseAPIMedia(t *testing.T) {
}
image := scraper.ParseAPIMedia(apimedia)
expected_id := 1395882862289772553
if image.ID != scraper.ImageID(expected_id) {
t.Errorf("Expected ID of %q, got %q", expected_id, image.ID)
}
expected_remote_url := "https://pbs.twimg.com/media/E18sEUrWYAk8dBl.jpg"
if image.RemoteURL != expected_remote_url {
t.Errorf("Expected %q, got %q", expected_remote_url, image.RemoteURL)