diff --git a/persistence/schema.sql b/persistence/schema.sql index 85a9956..70b8044 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -51,6 +51,7 @@ create table tweets (rowid integer primary key, mentions text, -- comma-separated reply_mentions text, -- comma-separated hashtags text, -- comma-separated + space_id text, tombstone_type integer default 0, is_stub boolean default 0, @@ -58,6 +59,7 @@ create table tweets (rowid integer primary key, is_conversation_scraped boolean default 0, last_scraped_at integer not null default 0, foreign key(user_id) references users(id) + foreign key(space_id) references spaces(id) ); create table retweets(rowid integer primary key, diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go index c199ad9..ee11e15 100644 --- a/persistence/tweet_queries.go +++ b/persistence/tweet_queries.go @@ -14,11 +14,20 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { tx := db.MustBegin() + var space_id scraper.SpaceID + for _, space := range t.Spaces { + err := p.SaveSpace(space) + if err != nil { + return err + } + space_id = space.ID + } + _, err := db.Exec(` insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, - quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded, + quoted_tweet_id, mentions, reply_mentions, hashtags, space_id, tombstone_type, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?, ?, ?) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, nullif(?, ''), (select rowid from tombstone_types where short_name=?), ?, ?, ?, ?) on conflict do update set text=(case when is_stub then @@ -45,7 +54,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { `, t.ID, t.UserID, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), - strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt, + strings.Join(t.Hashtags, ","), space_id, t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt, t.Text, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.TombstoneType, t.TombstoneType, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt, @@ -112,7 +121,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { stmt, err := db.Prepare(` select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded, + mentions, reply_mentions, hashtags, ifnull(space_id, ''), ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid where id = ? @@ -127,10 +136,11 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { var mentions string var reply_mentions string var hashtags string + var space_id scraper.SpaceID row := stmt.QueryRow(id) err = row.Scan(&t.ID, &t.UserID, &t.Text, &t.PostedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, - &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, + &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &space_id, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, &t.IsConversationScraped, &t.LastScrapedAt) if err != nil { return t, fmt.Errorf("Error parsing result in GetTweetByID(%d):\n %w", id, err) @@ -155,6 +165,15 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { } } + t.Spaces = []scraper.Space{} + if space_id != "" { + space, err := p.GetSpace(space_id) + if err != nil { + return t, err + } + t.Spaces = append(t.Spaces, space) + } + imgs, err := p.GetImagesForTweet(t) if err != nil { return t, err diff --git a/persistence/utils_test.go b/persistence/utils_test.go index d4b23d9..ec9dffe 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -171,6 +171,9 @@ func create_stable_tweet() scraper.Tweet { Polls: []scraper.Poll{ create_poll_from_id(-1), }, + Spaces: []scraper.Space{ + create_space_from_id(-1), + }, IsConversationScraped: true, LastScrapedAt: scraper.TimestampFromUnix(100000000), } @@ -254,6 +257,7 @@ func create_dummy_tweet() scraper.Tweet { ReplyMentions: []scraper.UserHandle{"replymention1", "replymention2"}, Hashtags: []string{"hash1", "hash2"}, Polls: []scraper.Poll{poll}, + Spaces: []scraper.Space{create_space_from_id(rand.Int())}, } } @@ -272,6 +276,7 @@ func create_dummy_tombstone() scraper.Tweet { Mentions: []scraper.UserHandle{}, ReplyMentions: []scraper.UserHandle{}, Hashtags: []string{}, + Spaces: []scraper.Space{}, } }