me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add persistence for reply-mentions

Browse Source
This commit is contained in:
Alessio 2021-09-27 18:29:55 -07:00
parent 159084006d
commit 6aa14c32e2
4 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tests.sh
View File

@ -81,6 +81,8 @@ test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_i
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice"
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730") = "kanesays23" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730") = "kanesays23"
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice"
# Test that the `--profile` flag works # Test that the `--profile` flag works

1
persistence/schema.sql
View File

@ -35,6 +35,7 @@ create table tweets (rowid integer primary key,
in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id` in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id`
quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id` quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id`
mentions text, -- comma-separated mentions text, -- comma-separated
reply_mentions text, -- comma-separated
hashtags text, -- comma-separated hashtags text, -- comma-separated
is_content_downloaded boolean default 0, is_content_downloaded boolean default 0,

14
persistence/tweet_queries.go
View File

@ -16,8 +16,8 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
return err return err
} }
_, err = db.Exec(` _, err = db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded) insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set num_likes=?, set num_likes=?,
num_retweets=?, num_retweets=?,
@ -25,7 +25,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
num_quote_tweets=?, num_quote_tweets=?,
is_content_downloaded=? is_content_downloaded=?
`, `,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded, t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded,
) )
@ -83,7 +83,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded
from tweets from tweets
where id = ? where id = ?
`) `)
@ -96,10 +96,11 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
var t scraper.Tweet var t scraper.Tweet
var postedAt int var postedAt int
var mentions string var mentions string
var reply_mentions string
var hashtags string var hashtags string
row := stmt.QueryRow(id) row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &hashtags, &t.IsContentDownloaded) err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.IsContentDownloaded)
if err != nil { if err != nil {
return t, err return t, err
} }
@ -108,6 +109,9 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
for _, m := range strings.Split(mentions, ",") { for _, m := range strings.Split(mentions, ",") {
t.Mentions = append(t.Mentions, scraper.UserHandle(m)) t.Mentions = append(t.Mentions, scraper.UserHandle(m))
} }
for _, m := range strings.Split(reply_mentions, ",") {
t.ReplyMentions = append(t.ReplyMentions, scraper.UserHandle(m))
}
t.Hashtags = strings.Split(hashtags, ",") t.Hashtags = strings.Split(hashtags, ",")
imgs, err := p.GetImagesForTweet(t) imgs, err := p.GetImagesForTweet(t)

1
persistence/utils_test.go
View File

@ -213,6 +213,7 @@ func create_dummy_tweet() scraper.Tweet {
Urls: []scraper.Url{url1, url2}, Urls: []scraper.Url{url1, url2},
Images: []scraper.Image{img1, img2}, Images: []scraper.Image{img1, img2},
Mentions: []scraper.UserHandle{"mention1", "mention2"}, Mentions: []scraper.UserHandle{"mention1", "mention2"},
ReplyMentions: []scraper.UserHandle{"replymention1", "replymention2"},
Hashtags: []string{"hash1", "hash2"}, Hashtags: []string{"hash1", "hash2"},
} }
} }