me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save links to Spaces in a Tweet

Browse Source
This commit is contained in:
Alessio 2022-05-14 16:36:03 -07:00
parent 3b6c970b74
commit cf7608eaa9
3 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence/schema.sql
View File

@ -51,6 +51,7 @@ create table tweets (rowid integer primary key,
mentions text, -- comma-separated
reply_mentions text, -- comma-separated
hashtags text, -- comma-separated
space_id text,
tombstone_type integer default 0,
is_stub boolean default 0,
@ -58,6 +59,7 @@ create table tweets (rowid integer primary key,
is_conversation_scraped boolean default 0,
last_scraped_at integer not null default 0,
foreign key(user_id) references users(id)
foreign key(space_id) references spaces(id)
);
create table retweets(rowid integer primary key,

29
persistence/tweet_queries.go
View File

@ -14,11 +14,20 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
tx := db.MustBegin()
var space_id scraper.SpaceID
for _, space := range t.Spaces {
err := p.SaveSpace(space)
if err != nil {
return err
}
space_id = space.ID
}
_, err := db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded,
quoted_tweet_id, mentions, reply_mentions, hashtags, space_id, tombstone_type, is_stub, is_content_downloaded,
is_conversation_scraped, last_scraped_at)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?, ?, ?)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, nullif(?, ''), (select rowid from tombstone_types where short_name=?), ?, ?, ?, ?)
on conflict do update
set text=(case
when is_stub then
@ -45,7 +54,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
`,
t.ID, t.UserID, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID,
t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions),
strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt,
strings.Join(t.Hashtags, ","), space_id, t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt,
t.Text, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.TombstoneType, t.TombstoneType,
t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt,
@ -112,7 +121,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded,
mentions, reply_mentions, hashtags, ifnull(space_id, ''), ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded,
is_conversation_scraped, last_scraped_at
from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where id = ?
@ -127,10 +136,11 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
var mentions string
var reply_mentions string
var hashtags string
var space_id scraper.SpaceID
row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &t.PostedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID,
&t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded,
&t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &space_id, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded,
&t.IsConversationScraped, &t.LastScrapedAt)
if err != nil {
return t, fmt.Errorf("Error parsing result in GetTweetByID(%d):\n %w", id, err)
@ -155,6 +165,15 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
}
}
t.Spaces = []scraper.Space{}
if space_id != "" {
space, err := p.GetSpace(space_id)
if err != nil {
return t, err
}
t.Spaces = append(t.Spaces, space)
}
imgs, err := p.GetImagesForTweet(t)
if err != nil {
return t, err

5
persistence/utils_test.go
View File

@ -171,6 +171,9 @@ func create_stable_tweet() scraper.Tweet {
Polls: []scraper.Poll{
create_poll_from_id(-1),
},
Spaces: []scraper.Space{
create_space_from_id(-1),
},
IsConversationScraped: true,
LastScrapedAt: scraper.TimestampFromUnix(100000000),
}
@ -254,6 +257,7 @@ func create_dummy_tweet() scraper.Tweet {
ReplyMentions: []scraper.UserHandle{"replymention1", "replymention2"},
Hashtags: []string{"hash1", "hash2"},
Polls: []scraper.Poll{poll},
Spaces: []scraper.Space{create_space_from_id(rand.Int())},
}
}
@ -272,6 +276,7 @@ func create_dummy_tombstone() scraper.Tweet {
Mentions: []scraper.UserHandle{},
ReplyMentions: []scraper.UserHandle{},
Hashtags: []string{},
Spaces: []scraper.Space{},
}
}