me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix re-downloading making users get marked as content-not-downloaded

Browse Source
This commit is contained in:
Alessio 2021-10-09 18:58:52 -07:00
parent b117cbec7b
commit 1c2f356ee9
4 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tests.sh
View File

@ -109,8 +109,10 @@ cd data
# Test that fetching tweets with ID only (not full URL) works # Test that fetching tweets with ID only (not full URL) works
test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "0" # Check it's not already there test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "0" # Check it's not already there
test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='elonmusk'") = "1" # Should be downloaded from the previous test!
tw fetch_tweet 1433713164546293767 tw fetch_tweet 1433713164546293767
test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "1" # Should be there now test $(sqlite3 twitter.db "select count(*) from tweets where id = 1433713164546293767") = "1" # Should be there now
test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='elonmusk'") = "1" # Should not un-set content-downloaded!
# Get a user's feed # Get a user's feed

6
persistence/media_queries.go
View File

@ -15,7 +15,7 @@ func (p Profile) SaveImage(img scraper.Image) error {
insert into images (id, tweet_id, remote_url, local_filename, is_downloaded) insert into images (id, tweet_id, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?) values (?, ?, ?, ?, ?)
on conflict do update on conflict do update
set is_downloaded=? set is_downloaded=(is_downloaded or ?)
`, `,
img.ID, img.TweetID, img.RemoteURL, img.LocalFilename, img.IsDownloaded, img.ID, img.TweetID, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
img.IsDownloaded, img.IsDownloaded,
@ -34,7 +34,7 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif) insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif)
values (?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set is_downloaded=? set is_downloaded=(is_downloaded or ?)
`, `,
vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif, vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif,
vid.IsDownloaded, vid.IsDownloaded,
@ -50,7 +50,7 @@ func (p Profile) SaveUrl(url scraper.Url) error {
insert into urls (tweet_id, domain, text, title, description, creator_id, site_id, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) insert into urls (tweet_id, domain, text, title, description, creator_id, site_id, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set is_content_downloaded=? set is_content_downloaded=(is_content_downloaded or ?)
`, `,
url.TweetID, url.Domain, url.Text, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded, url.TweetID, url.Domain, url.Text, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
url.IsContentDownloaded, url.IsContentDownloaded,

2
persistence/tweet_queries.go
View File

@ -23,7 +23,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
num_retweets=?, num_retweets=?,
num_replies=?, num_replies=?,
num_quote_tweets=?, num_quote_tweets=?,
is_content_downloaded=? is_content_downloaded=(is_content_downloaded or ?)
`, `,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded, t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded,

2
persistence/user_queries.go
View File

@ -32,7 +32,7 @@ func (p Profile) SaveUser(u scraper.User) error {
banner_image_url=?, banner_image_url=?,
banner_image_local_path=?, banner_image_local_path=?,
pinned_tweet_id=?, pinned_tweet_id=?,
is_content_downloaded=? is_content_downloaded=(is_content_downloaded or ?)
`, `,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,