me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix re-downloading making users get marked as content-not-downloaded

Browse Source
This commit is contained in:
Alessio 2021-10-09 18:58:52 -07:00
parent b117cbec7b
commit 1c2f356ee9
4 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tests.sh
View File

@ -109,8 +109,10 @@ cd data
# Test that fetching tweets with ID only (not full URL) works
test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "0" # Check it's not already there
test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='elonmusk'") = "1" # Should be downloaded from the previous test!
tw fetch_tweet 1433713164546293767
test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "1" # Should be there now
test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='elonmusk'") = "1" # Should not un-set content-downloaded!
# Get a user's feed

6
persistence/media_queries.go
View File

@ -15,7 +15,7 @@ func (p Profile) SaveImage(img scraper.Image) error {
insert into images (id, tweet_id, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=?
set is_downloaded=(is_downloaded or ?)
`,
img.ID, img.TweetID, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
img.IsDownloaded,
@ -34,7 +34,7 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif)
values (?, ?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=?
set is_downloaded=(is_downloaded or ?)
`,
vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif,
vid.IsDownloaded,
@ -50,7 +50,7 @@ func (p Profile) SaveUrl(url scraper.Url) error {
insert into urls (tweet_id, domain, text, title, description, creator_id, site_id, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update
set is_content_downloaded=?
set is_content_downloaded=(is_content_downloaded or ?)
`,
url.TweetID, url.Domain, url.Text, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
url.IsContentDownloaded,

2
persistence/tweet_queries.go
View File

@ -23,7 +23,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
num_retweets=?,
num_replies=?,
num_quote_tweets=?,
is_content_downloaded=?
is_content_downloaded=(is_content_downloaded or ?)
`,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded,

2
persistence/user_queries.go
View File

@ -32,7 +32,7 @@ func (p Profile) SaveUser(u scraper.User) error {
banner_image_url=?,
banner_image_local_path=?,
pinned_tweet_id=?,
is_content_downloaded=?
is_content_downloaded=(is_content_downloaded or ?)
`,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,