From 14b9143f760bf74d8701d116267aee7e3fb1b94e Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 4 Sep 2023 13:11:58 -0300 Subject: [PATCH] Compound db queries now fetch whether the tweet is liked by the current logged-in user --- internal/webserver/handler_search.go | 2 +- internal/webserver/handler_timeline.go | 2 +- internal/webserver/handler_tweet_detail.go | 2 +- internal/webserver/handler_user_feed.go | 2 +- internal/webserver/static/icons/quote.svg | 9 ++--- pkg/persistence/compound_queries.go | 22 +++++++----- pkg/persistence/compound_queries_test.go | 36 +++++++++++++++----- pkg/persistence/compound_ssf_queries.go | 13 ++++--- pkg/persistence/compound_ssf_queries_test.go | 36 ++++++++++---------- pkg/scraper/tweet.go | 1 + sample_data/seed_data.sql | 1 + 11 files changed, 76 insertions(+), 50 deletions(-) diff --git a/internal/webserver/handler_search.go b/internal/webserver/handler_search.go index 12be492..f69333b 100644 --- a/internal/webserver/handler_search.go +++ b/internal/webserver/handler_search.go @@ -66,7 +66,7 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) { return } - feed, err := app.Profile.NextPage(c) + feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) if err != nil { if errors.Is(err, persistence.ErrEndOfFeed) { // TODO diff --git a/internal/webserver/handler_timeline.go b/internal/webserver/handler_timeline.go index 4152c56..ca56b51 100644 --- a/internal/webserver/handler_timeline.go +++ b/internal/webserver/handler_timeline.go @@ -17,7 +17,7 @@ func (app *Application) Timeline(w http.ResponseWriter, r *http.Request) { return } - feed, err := app.Profile.NextPage(c) + feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) if err != nil { if errors.Is(err, persistence.ErrEndOfFeed) { // TODO diff --git a/internal/webserver/handler_tweet_detail.go b/internal/webserver/handler_tweet_detail.go index 3dbcfe6..a4a8121 100644 --- a/internal/webserver/handler_tweet_detail.go +++ b/internal/webserver/handler_tweet_detail.go @@ -79,7 +79,7 @@ func (app *Application) TweetDetail(w http.ResponseWriter, r *http.Request) { try_scrape_tweet() // If it fails, we can still render it (not 404) } - trove, err := app.Profile.GetTweetDetail(data.MainTweetID) + trove, err := app.Profile.GetTweetDetail(data.MainTweetID, app.ActiveUser.ID) if err != nil { if errors.Is(err, persistence.ErrNotInDB) { app.error_404(w) diff --git a/internal/webserver/handler_user_feed.go b/internal/webserver/handler_user_feed.go index 1069d4d..df663dc 100644 --- a/internal/webserver/handler_user_feed.go +++ b/internal/webserver/handler_user_feed.go @@ -63,7 +63,7 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { return } - feed, err := app.Profile.NextPage(c) + feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) if err != nil { if errors.Is(err, persistence.ErrEndOfFeed) { // TODO diff --git a/internal/webserver/static/icons/quote.svg b/internal/webserver/static/icons/quote.svg index 4e4eaf3..3b18086 100644 --- a/internal/webserver/static/icons/quote.svg +++ b/internal/webserver/static/icons/quote.svg @@ -1,7 +1,2 @@ - - - - - - - \ No newline at end of file + + \ No newline at end of file diff --git a/pkg/persistence/compound_queries.go b/pkg/persistence/compound_queries.go index 61243c5..7639025 100644 --- a/pkg/persistence/compound_queries.go +++ b/pkg/persistence/compound_queries.go @@ -14,12 +14,13 @@ var ( ) const TWEETS_ALL_SQL_FIELDS = ` - tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, + tweets.id id, tweets.user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text, + case when likes.user_id is null then 0 else 1 end is_liked_by_current_user, is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at` -func (p Profile) fill_content(trove *TweetTrove) { +func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { if len(trove.Tweets) == 0 { // Empty trove, nothing to fetch return @@ -39,7 +40,8 @@ func (p Profile) fill_content(trove *TweetTrove) { select `+TWEETS_ALL_SQL_FIELDS+` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid - where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, quoted_ids...) + left join likes on tweets.id = likes.tweet_id and likes.user_id = ? + where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, append([]interface{}{current_user_id}, quoted_ids...)...) if err != nil { panic(err) } @@ -207,7 +209,7 @@ func NewTweetDetailView() TweetDetailView { } // Return the given tweet, all its parent tweets, and a list of conversation threads -func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { +func (p Profile) GetTweetDetail(id TweetID, current_user_id UserID) (TweetDetailView, error) { // TODO: compound-query-structs ret := NewTweetDetailView() ret.MainTweetID = id @@ -221,6 +223,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { select ` + TWEETS_ALL_SQL_FIELDS + ` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = ? inner join all_replies on tweets.id = all_replies.id order by id asc`) if err != nil { @@ -230,7 +233,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { // Main tweet and parents var thread []Tweet - err = stmt.Select(&thread, id) + err = stmt.Select(&thread, id, current_user_id) if err != nil { panic(err) } @@ -249,6 +252,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { `select ` + TWEETS_ALL_SQL_FIELDS + ` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = ? where in_reply_to_id = ? order by num_likes desc limit 50`) @@ -256,7 +260,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { panic(err) } defer stmt.Close() - err = stmt.Select(&replies, id) + err = stmt.Select(&replies, current_user_id, id) if err != nil { panic(err) } @@ -287,7 +291,9 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { select ` + TWEETS_ALL_SQL_FIELDS + ` from top_ids_by_parent left join tweets on tweets.id = top_ids_by_parent.id - left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid` + left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = ?` + reply_1_ids = append(reply_1_ids, current_user_id) err = p.DB.Select(&replies, reply2_query, reply_1_ids...) if err != nil { panic(err) @@ -304,7 +310,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { } } - p.fill_content(&ret.TweetTrove) + p.fill_content(&ret.TweetTrove, current_user_id) return ret, nil } diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index 47fda40..9e68fb4 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -21,7 +21,7 @@ func TestBuildUserFeed(t *testing.T) { c := persistence.NewUserFeedCursor(UserHandle("cernovich")) c.PageSize = 2 - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Retweets, 2) @@ -63,7 +63,7 @@ func TestBuildUserFeedPage2(t *testing.T) { c.PageSize = 2 c.CursorPosition = persistence.CURSOR_MIDDLE c.CursorValue = 1644107102 - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Retweets, 1) @@ -103,7 +103,7 @@ func TestBuildUserFeedEnd(t *testing.T) { c.PageSize = 2 c.CursorPosition = persistence.CURSOR_MIDDLE c.CursorValue = 1 // Won't be anything - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Retweets, 0) @@ -114,6 +114,24 @@ func TestBuildUserFeedEnd(t *testing.T) { assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END) } +func TestUserFeedHasLikesInfo(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Fetch @Peter_Nimitz user feed while logged in as @MysteryGrove + c := persistence.NewUserFeedCursor(UserHandle("Peter_Nimitz")) + feed, err := profile.NextPage(c, UserID(1178839081222115328)) + require.NoError(err) + + // Should have "liked" 1 tweet + for _, t := range feed.Tweets { + assert.Equal(t.IsLikedByCurrentUser, t.ID == TweetID(1413646595493568516)) + } +} + func TestUserFeedWithTombstone(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -122,7 +140,7 @@ func TestUserFeedWithTombstone(t *testing.T) { require.NoError(err) c := persistence.NewUserFeedCursor(UserHandle("Heminator")) - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) tombstone_tweet := feed.Tweets[TweetID(31)] assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author") @@ -135,7 +153,7 @@ func TestTweetDetailWithReplies(t *testing.T) { profile, err := persistence.LoadProfile("../../sample_data/profile") require.NoError(err) - tweet_detail, err := profile.GetTweetDetail(TweetID(1413646595493568516)) + tweet_detail, err := profile.GetTweetDetail(TweetID(1413646595493568516), UserID(1178839081222115328)) require.NoError(err) assert.Len(tweet_detail.Retweets, 0) @@ -151,8 +169,9 @@ func TestTweetDetailWithReplies(t *testing.T) { 1413772782358433792, 1413773185296650241, } { - _, is_ok := tweet_detail.Tweets[id] + t, is_ok := tweet_detail.Tweets[id] assert.True(is_ok) + assert.Equal(t.IsLikedByCurrentUser, id == 1413646595493568516) } assert.Len(tweet_detail.Users, 4) @@ -189,7 +208,7 @@ func TestTweetDetailWithParents(t *testing.T) { profile, err := persistence.LoadProfile("../../sample_data/profile") require.NoError(err) - tweet_detail, err := profile.GetTweetDetail(TweetID(1413773185296650241)) + tweet_detail, err := profile.GetTweetDetail(TweetID(1413773185296650241), UserID(1178839081222115328)) require.NoError(err) assert.Len(tweet_detail.Retweets, 0) @@ -201,8 +220,9 @@ func TestTweetDetailWithParents(t *testing.T) { 1413772782358433792, 1413773185296650241, } { - _, is_ok := tweet_detail.Tweets[id] + t, is_ok := tweet_detail.Tweets[id] assert.True(is_ok) + assert.Equal(t.IsLikedByCurrentUser, id == 1413646595493568516) } assert.Len(tweet_detail.Users, 2) diff --git a/pkg/persistence/compound_ssf_queries.go b/pkg/persistence/compound_ssf_queries.go index 3255b46..8ce5576 100644 --- a/pkg/persistence/compound_ssf_queries.go +++ b/pkg/persistence/compound_ssf_queries.go @@ -261,7 +261,7 @@ func (c *Cursor) apply_token(token string) error { return nil } -func (p Profile) NextPage(c Cursor) (Feed, error) { +func (p Profile) NextPage(c Cursor, current_user_id scraper.UserID) (Feed, error) { where_clauses := []string{} bind_values := []interface{}{} @@ -273,7 +273,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { // From, to, by, and RT'd by user handles if c.FromUserHandle != "" { - where_clauses = append(where_clauses, "user_id = (select id from users where handle like ?)") + where_clauses = append(where_clauses, "tweets.user_id = (select id from users where handle like ?)") bind_values = append(bind_values, c.FromUserHandle) } for _, to_user := range c.ToUserHandles { @@ -362,9 +362,10 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { q := `select * from ( select ` + TWEETS_ALL_SQL_FIELDS + `, 0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at, - posted_at chrono, user_id by_user_id + posted_at chrono, tweets.user_id by_user_id from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = ? ` + where_clause + ` ` + c.SortOrder.OrderByClause() + ` limit ? ) @@ -372,16 +373,18 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { select * from ( select ` + TWEETS_ALL_SQL_FIELDS + `, - tweet_id, retweet_id, retweeted_by, retweeted_at, + retweets.tweet_id, retweet_id, retweeted_by, retweeted_at, retweeted_at chrono, retweeted_by by_user_id from retweets left join tweets on retweets.tweet_id = tweets.id left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = ? ` + where_clause + ` ` + c.SortOrder.OrderByClause() + ` limit ? ) ` + c.SortOrder.OrderByClause() + ` limit ?` + bind_values = append([]interface{}{current_user_id}, bind_values...) bind_values = append(bind_values, c.PageSize) bind_values = append(bind_values, bind_values...) bind_values = append(bind_values, c.PageSize) @@ -406,7 +409,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID}) } - p.fill_content(&ret.TweetTrove) + p.fill_content(&ret.TweetTrove, current_user_id) ret.CursorBottom = c diff --git a/pkg/persistence/compound_ssf_queries_test.go b/pkg/persistence/compound_ssf_queries_test.go index faa0e17..0daaf38 100644 --- a/pkg/persistence/compound_ssf_queries_test.go +++ b/pkg/persistence/compound_ssf_queries_test.go @@ -25,7 +25,7 @@ func TestCursorSearchByNewest(t *testing.T) { c.Keywords = []string{"think"} c.SortOrder = persistence.SORT_ORDER_NEWEST - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 3) @@ -41,7 +41,7 @@ func TestCursorSearchByNewest(t *testing.T) { assert.Equal(next_cursor.PageSize, c.PageSize) assert.Equal(next_cursor.CursorValue, 1629520619) - feed, err = profile.NextPage(next_cursor) + feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) assert.Len(feed.Items, 2) @@ -67,7 +67,7 @@ func TestCursorSearchWithRetweets(t *testing.T) { c.FilterRetweets = persistence.REQUIRE c.SortOrder = persistence.SORT_ORDER_OLDEST - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 3) @@ -83,7 +83,7 @@ func TestCursorSearchWithRetweets(t *testing.T) { assert.Equal(next_cursor.PageSize, c.PageSize) assert.Equal(next_cursor.CursorValue, 1644111031) - feed, err = profile.NextPage(next_cursor) + feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) assert.Len(feed.Items, 0) @@ -102,7 +102,7 @@ func TestTimeline(t *testing.T) { c := persistence.NewTimelineCursor() c.PageSize = 5 - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 5) @@ -121,7 +121,7 @@ func TestTimeline(t *testing.T) { assert.Equal(next_cursor.CursorValue, 1635367140) next_cursor.CursorValue = 1631935323 // Scroll down a bit, kind of randomly - feed, err = profile.NextPage(next_cursor) + feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) assert.Len(feed.Items, 5) @@ -143,20 +143,20 @@ func TestKeywordSearch(t *testing.T) { // Multiple words without quotes c.Keywords = []string{"who", "are"} - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.True(len(feed.Items) > 1) // Add quotes c.Keywords = []string{"who are"} - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 1) assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700)) // With gibberish (no matches) c.Keywords = []string{"fasdfjkafsldfjsff"} - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 0) } @@ -171,7 +171,7 @@ func TestSearchReplyingToUser(t *testing.T) { // Replying to a user c.ToUserHandles = []UserHandle{"spacex"} - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 2) assert.Equal(feed.Items[0].TweetID, TweetID(1428951883058753537)) @@ -179,7 +179,7 @@ func TestSearchReplyingToUser(t *testing.T) { // Replying to two users c.ToUserHandles = []UserHandle{"spacex", "covfefeanon"} - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 1) assert.Equal(feed.Items[0].TweetID, TweetID(1428939163961790466)) @@ -197,7 +197,7 @@ func TestSearchDateFilters(t *testing.T) { // Since timestamp c.SinceTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC) c.FromUserHandle = UserHandle("cernovich") - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 1) assert.Equal(feed.Items[0].TweetID, TweetID(1453461248142495744)) @@ -205,7 +205,7 @@ func TestSearchDateFilters(t *testing.T) { // Until timestamp c.SinceTimestamp = TimestampFromUnix(0) c.UntilTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC) - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 3) assert.Equal(feed.Items[0].TweetID, TweetID(1439027915404939265)) @@ -224,7 +224,7 @@ func TestSearchMediaFilters(t *testing.T) { c := persistence.NewCursor() c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.FilterLinks = persistence.REQUIRE - feed, err := profile.NextPage(c) + feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 2) assert.Equal(feed.Items[0].TweetID, TweetID(1438642143170646017)) @@ -234,7 +234,7 @@ func TestSearchMediaFilters(t *testing.T) { c = persistence.NewCursor() c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.FilterImages = persistence.REQUIRE - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 2) assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700)) @@ -244,7 +244,7 @@ func TestSearchMediaFilters(t *testing.T) { c = persistence.NewCursor() c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.FilterVideos = persistence.REQUIRE - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 2) assert.Equal(feed.Items[0].TweetID, TweetID(1426619468327882761)) @@ -253,7 +253,7 @@ func TestSearchMediaFilters(t *testing.T) { // Polls c = persistence.NewCursor() c.FilterPolls = persistence.REQUIRE - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 1) assert.Equal(feed.Items[0].TweetID, TweetID(1465534109573390348)) @@ -261,7 +261,7 @@ func TestSearchMediaFilters(t *testing.T) { // Spaces c = persistence.NewCursor() c.FilterSpaces = persistence.REQUIRE - feed, err = profile.NextPage(c) + feed, err = profile.NextPage(c, UserID(0)) require.NoError(err) assert.Len(feed.Items, 1) assert.Equal(feed.Items[0].TweetID, TweetID(1624833173514293249)) diff --git a/pkg/scraper/tweet.go b/pkg/scraper/tweet.go index 86405c8..65020c4 100644 --- a/pkg/scraper/tweet.go +++ b/pkg/scraper/tweet.go @@ -73,6 +73,7 @@ type Tweet struct { TombstoneText string `db:"tombstone_text"` IsStub bool `db:"is_stub"` + IsLikedByCurrentUser bool `db:"is_liked_by_current_user"` IsContentDownloaded bool `db:"is_content_downloaded"` IsConversationScraped bool `db:"is_conversation_scraped"` LastScrapedAt Timestamp `db:"last_scraped_at"` diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index 1335daf..ddcd205 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -287,6 +287,7 @@ create table likes(rowid integer primary key, foreign key(user_id) references users(id) foreign key(tweet_id) references tweets(id) ); +insert into likes values(1, 1, 1178839081222115328, 1413646595493568516); create table fake_user_sequence(latest_fake_id integer not null); insert into fake_user_sequence values(0x4000000000000000);