me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add persistence for reply-mentions

Browse Source
This commit is contained in:
Alessio 2021-09-27 18:29:55 -07:00
parent 159084006d
commit 6aa14c32e2
4 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tests.sh
View File

@ -81,6 +81,8 @@ test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_i
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice"
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730") = "kanesays23"
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice"
# Test that the `--profile` flag works

5
persistence/schema.sql
View File

@ -34,8 +34,9 @@ create table tweets (rowid integer primary key,
num_quote_tweets integer,
in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id`
quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id`
mentions text, -- comma-separated
hashtags text, -- comma-separated
mentions text, -- comma-separated
reply_mentions text, -- comma-separated
hashtags text, -- comma-separated
is_content_downloaded boolean default 0,
foreign key(user_id) references users(id)

14
persistence/tweet_queries.go
View File

@ -16,8 +16,8 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
return err
}
_, err = db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update
set num_likes=?,
num_retweets=?,
@ -25,7 +25,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
num_quote_tweets=?,
is_content_downloaded=?
`,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded,
)
@ -83,7 +83,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
db := p.DB
stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded
from tweets
where id = ?
`)
@ -96,10 +96,11 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
var t scraper.Tweet
var postedAt int
var mentions string
var reply_mentions string
var hashtags string
row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &hashtags, &t.IsContentDownloaded)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.IsContentDownloaded)
if err != nil {
return t, err
}
@ -108,6 +109,9 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
for _, m := range strings.Split(mentions, ",") {
t.Mentions = append(t.Mentions, scraper.UserHandle(m))
}
for _, m := range strings.Split(reply_mentions, ",") {
t.ReplyMentions = append(t.ReplyMentions, scraper.UserHandle(m))
}
t.Hashtags = strings.Split(hashtags, ",")
imgs, err := p.GetImagesForTweet(t)

1
persistence/utils_test.go
View File

@ -213,6 +213,7 @@ func create_dummy_tweet() scraper.Tweet {
Urls: []scraper.Url{url1, url2},
Images: []scraper.Image{img1, img2},
Mentions: []scraper.UserHandle{"mention1", "mention2"},
ReplyMentions: []scraper.UserHandle{"replymention1", "replymention2"},
Hashtags: []string{"hash1", "hash2"},
}
}