me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compound db queries now fetch whether the tweet is liked by the current logged-in user

Browse Source
This commit is contained in:
Alessio 2023-09-04 13:11:58 -03:00
parent 3e9bc9f4f3
commit 14b9143f76
11 changed files with 76 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/webserver/handler_search.go
View File

@ -66,7 +66,7 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) {
return return
} }
feed, err := app.Profile.NextPage(c) feed, err := app.Profile.NextPage(c, app.ActiveUser.ID)
if err != nil { if err != nil {
if errors.Is(err, persistence.ErrEndOfFeed) { if errors.Is(err, persistence.ErrEndOfFeed) {
// TODO // TODO

2
internal/webserver/handler_timeline.go
View File

@ -17,7 +17,7 @@ func (app *Application) Timeline(w http.ResponseWriter, r *http.Request) {
return return
} }
feed, err := app.Profile.NextPage(c) feed, err := app.Profile.NextPage(c, app.ActiveUser.ID)
if err != nil { if err != nil {
if errors.Is(err, persistence.ErrEndOfFeed) { if errors.Is(err, persistence.ErrEndOfFeed) {
// TODO // TODO

2
internal/webserver/handler_tweet_detail.go
View File

@ -79,7 +79,7 @@ func (app *Application) TweetDetail(w http.ResponseWriter, r *http.Request) {
try_scrape_tweet() // If it fails, we can still render it (not 404) try_scrape_tweet() // If it fails, we can still render it (not 404)
} }
trove, err := app.Profile.GetTweetDetail(data.MainTweetID) trove, err := app.Profile.GetTweetDetail(data.MainTweetID, app.ActiveUser.ID)
if err != nil { if err != nil {
if errors.Is(err, persistence.ErrNotInDB) { if errors.Is(err, persistence.ErrNotInDB) {
app.error_404(w) app.error_404(w)

2
internal/webserver/handler_user_feed.go
View File

@ -63,7 +63,7 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
return return
} }
feed, err := app.Profile.NextPage(c) feed, err := app.Profile.NextPage(c, app.ActiveUser.ID)
if err != nil { if err != nil {
if errors.Is(err, persistence.ErrEndOfFeed) { if errors.Is(err, persistence.ErrEndOfFeed) {
// TODO // TODO

9
internal/webserver/static/icons/quote.svg
View File

@ -1,7 +1,2 @@
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools --><svg fill="#000000" height="800px" width="800px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-3.2 -3.2 70.40 70.40" enable-background="new 0 0 64 64" xml:space="preserve" stroke="#000000" stroke-width="3.2"><g id="SVGRepo_bgCarrier" stroke-width="0"/><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/><g id="SVGRepo_iconCarrier"> <g id="Quotemarks-right"> <path d="M14.1933422,9.4116497c-7.8260994,0-14.1922989,6.3662004-14.1922989,14.1924 c0,7.5498009,5.9247999,13.7420998,13.3690996,14.169899c0.1288996,1.3916016,0.0321999,5.1797028-3.5977001,10.4491997 C9.4980431,48.6206512,9.547843,49.1567497,9.888648,49.497551c1.4853945,1.4853973,2.4033947,2.4208984,3.0458946,3.0751991 c0.8408995,0.8554993,1.2247,1.2461014,1.7861996,1.7559013c0.1904001,0.1727982,0.4306002,0.259697,0.6719055,0.259697 c0.2342949,0,0.4676943-0.0819969,0.6561956-0.2450981c6.3251991-5.5038986,13.3515987-16.8759995,12.3349991-30.8115005 C27.7881413,15.3501501,21.820343,9.4116497,14.1933422,9.4116497z M15.4023428,52.2221489 c-0.2723999-0.2684975-0.5830002-0.5848999-1.0410004-1.0508003c-0.5565996-0.5672989-1.3203001-1.3446999-2.4784994-2.5067978 c4.4053001-6.7881012,3.5731993-11.6230011,3.2089996-12.3164024c-0.1729002-0.3290977-0.5274-0.5507965-0.8985004-0.5507965 c-6.7225995,0-12.1922989-5.4697018-12.1922989-12.1933022c0-6.7227001,5.4696999-12.1924,12.1922989-12.1924 c6.5489006,0,11.6777992,5.1582012,12.1963062,12.2646008C27.5322418,39.3501511,18.2168427,49.5268517,15.4023428,52.2221489z"/> <path d="M63.9004402,23.5317497v-0.0009995C63.302742,15.3501501,57.3340416,9.4116497,49.7090416,9.4116497 c-7.8261986,0-14.1933937,6.3662004-14.1933937,14.1924c0,7.5498009,5.9257927,13.7420998,13.3710938,14.169899 c0.1289062,1.3906021,0.0312004,5.1767006-3.5996017,10.4491997c-0.2743988,0.3975029-0.2245979,0.9336014,0.1162033,1.2744026 c1.4794998,1.4794998,2.3955002,2.4130974,3.0380974,3.0663986c0.8446999,0.8613014,1.2304993,1.2538986,1.7949028,1.7656021 c0.1903992,0.1718979,0.4315987,0.2587967,0.6718979,0.2587967c0.2344055,0,0.4678001-0.0819969,0.6562004-0.2460976 C57.8896484,48.8383484,64.9160385,37.4663506,63.9004402,23.5317497z M50.917942,52.2221489 c-0.2743988-0.2705002-0.5877991-0.5887985-1.0498009-1.0594978c-0.5565987-0.5665016-1.3172989-1.3418007-2.4706993-2.4981003 c4.4053001-6.7891006,3.5742989-11.6230011,3.2109985-12.3164024c-0.1728973-0.3280983-0.5282974-0.5507965-0.8993988-0.5507965 c-6.7237015,0-12.1933937-5.4697018-12.1933937-12.1933022c0-6.7227001,5.4696922-12.1924,12.1933937-12.1924 c6.5477982,0,11.6777,5.1582012,12.1972008,12.2656002v-0.0009995 C63.0478401,39.3481483,53.7324409,49.5268517,50.917942,52.2221489z"/> </g> </g></svg>
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
<svg fill="#000000" height="800px" width="800px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-3.2 -3.2 70.40 70.40" enable-background="new 0 0 64 64" xml:space="preserve" stroke="#000000" stroke-width="3.2">
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
<g id="SVGRepo_iconCarrier"> <g id="Quotemarks-right"> <path d="M14.1933422,9.4116497c-7.8260994,0-14.1922989,6.3662004-14.1922989,14.1924 c0,7.5498009,5.9247999,13.7420998,13.3690996,14.169899c0.1288996,1.3916016,0.0321999,5.1797028-3.5977001,10.4491997 C9.4980431,48.6206512,9.547843,49.1567497,9.888648,49.497551c1.4853945,1.4853973,2.4033947,2.4208984,3.0458946,3.0751991 c0.8408995,0.8554993,1.2247,1.2461014,1.7861996,1.7559013c0.1904001,0.1727982,0.4306002,0.259697,0.6719055,0.259697 c0.2342949,0,0.4676943-0.0819969,0.6561956-0.2450981c6.3251991-5.5038986,13.3515987-16.8759995,12.3349991-30.8115005 C27.7881413,15.3501501,21.820343,9.4116497,14.1933422,9.4116497z M15.4023428,52.2221489 c-0.2723999-0.2684975-0.5830002-0.5848999-1.0410004-1.0508003c-0.5565996-0.5672989-1.3203001-1.3446999-2.4784994-2.5067978 c4.4053001-6.7881012,3.5731993-11.6230011,3.2089996-12.3164024c-0.1729002-0.3290977-0.5274-0.5507965-0.8985004-0.5507965 c-6.7225995,0-12.1922989-5.4697018-12.1922989-12.1933022c0-6.7227001,5.4696999-12.1924,12.1922989-12.1924 c6.5489006,0,11.6777992,5.1582012,12.1963062,12.2646008C27.5322418,39.3501511,18.2168427,49.5268517,15.4023428,52.2221489z"/> <path d="M63.9004402,23.5317497v-0.0009995C63.302742,15.3501501,57.3340416,9.4116497,49.7090416,9.4116497 c-7.8261986,0-14.1933937,6.3662004-14.1933937,14.1924c0,7.5498009,5.9257927,13.7420998,13.3710938,14.169899 c0.1289062,1.3906021,0.0312004,5.1767006-3.5996017,10.4491997c-0.2743988,0.3975029-0.2245979,0.9336014,0.1162033,1.2744026 c1.4794998,1.4794998,2.3955002,2.4130974,3.0380974,3.0663986c0.8446999,0.8613014,1.2304993,1.2538986,1.7949028,1.7656021 c0.1903992,0.1718979,0.4315987,0.2587967,0.6718979,0.2587967c0.2344055,0,0.4678001-0.0819969,0.6562004-0.2460976 C57.8896484,48.8383484,64.9160385,37.4663506,63.9004402,23.5317497z M50.917942,52.2221489 c-0.2743988-0.2705002-0.5877991-0.5887985-1.0498009-1.0594978c-0.5565987-0.5665016-1.3172989-1.3418007-2.4706993-2.4981003 c4.4053001-6.7891006,3.5742989-11.6230011,3.2109985-12.3164024c-0.1728973-0.3280983-0.5282974-0.5507965-0.8993988-0.5507965 c-6.7237015,0-12.1933937-5.4697018-12.1933937-12.1933022c0-6.7227001,5.4696922-12.1924,12.1933937-12.1924 c6.5477982,0,11.6777,5.1582012,12.1972008,12.2656002v-0.0009995 C63.0478401,39.3481483,53.7324409,49.5268517,50.917942,52.2221489z"/> </g> </g>
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

22
pkg/persistence/compound_queries.go
View File

@ -14,12 +14,13 @@ var (
) )
const TWEETS_ALL_SQL_FIELDS = ` const TWEETS_ALL_SQL_FIELDS = `
tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, tweets.id id, tweets.user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text, ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text,
case when likes.user_id is null then 0 else 1 end is_liked_by_current_user,
is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at` is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at`
func (p Profile) fill_content(trove *TweetTrove) { func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) {
if len(trove.Tweets) == 0 { if len(trove.Tweets) == 0 {
// Empty trove, nothing to fetch // Empty trove, nothing to fetch
return return
@ -39,7 +40,8 @@ func (p Profile) fill_content(trove *TweetTrove) {
select `+TWEETS_ALL_SQL_FIELDS+` select `+TWEETS_ALL_SQL_FIELDS+`
from tweets from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, quoted_ids...) left join likes on tweets.id = likes.tweet_id and likes.user_id = ?
where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, append([]interface{}{current_user_id}, quoted_ids...)...)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -207,7 +209,7 @@ func NewTweetDetailView() TweetDetailView {
} }
// Return the given tweet, all its parent tweets, and a list of conversation threads // Return the given tweet, all its parent tweets, and a list of conversation threads
func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { func (p Profile) GetTweetDetail(id TweetID, current_user_id UserID) (TweetDetailView, error) {
// TODO: compound-query-structs // TODO: compound-query-structs
ret := NewTweetDetailView() ret := NewTweetDetailView()
ret.MainTweetID = id ret.MainTweetID = id
@ -221,6 +223,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
select ` + TWEETS_ALL_SQL_FIELDS + ` select ` + TWEETS_ALL_SQL_FIELDS + `
from tweets from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
left join likes on tweets.id = likes.tweet_id and likes.user_id = ?
inner join all_replies on tweets.id = all_replies.id inner join all_replies on tweets.id = all_replies.id
order by id asc`) order by id asc`)
if err != nil { if err != nil {
@ -230,7 +233,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
// Main tweet and parents // Main tweet and parents
var thread []Tweet var thread []Tweet
err = stmt.Select(&thread, id) err = stmt.Select(&thread, id, current_user_id)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -249,6 +252,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
`select ` + TWEETS_ALL_SQL_FIELDS + ` `select ` + TWEETS_ALL_SQL_FIELDS + `
from tweets from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
left join likes on tweets.id = likes.tweet_id and likes.user_id = ?
where in_reply_to_id = ? where in_reply_to_id = ?
order by num_likes desc order by num_likes desc
limit 50`) limit 50`)
@ -256,7 +260,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
panic(err) panic(err)
} }
defer stmt.Close() defer stmt.Close()
err = stmt.Select(&replies, id) err = stmt.Select(&replies, current_user_id, id)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -287,7 +291,9 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
select ` + TWEETS_ALL_SQL_FIELDS + ` select ` + TWEETS_ALL_SQL_FIELDS + `
from top_ids_by_parent from top_ids_by_parent
left join tweets on tweets.id = top_ids_by_parent.id left join tweets on tweets.id = top_ids_by_parent.id
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid` left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
left join likes on tweets.id = likes.tweet_id and likes.user_id = ?`
reply_1_ids = append(reply_1_ids, current_user_id)
err = p.DB.Select(&replies, reply2_query, reply_1_ids...) err = p.DB.Select(&replies, reply2_query, reply_1_ids...)
if err != nil { if err != nil {
panic(err) panic(err)
@ -304,7 +310,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
} }
} }
p.fill_content(&ret.TweetTrove) p.fill_content(&ret.TweetTrove, current_user_id)
return ret, nil return ret, nil
} }

36
pkg/persistence/compound_queries_test.go
View File

@ -21,7 +21,7 @@ func TestBuildUserFeed(t *testing.T) {
c := persistence.NewUserFeedCursor(UserHandle("cernovich")) c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
c.PageSize = 2 c.PageSize = 2
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Retweets, 2) assert.Len(feed.Retweets, 2)
@ -63,7 +63,7 @@ func TestBuildUserFeedPage2(t *testing.T) {
c.PageSize = 2 c.PageSize = 2
c.CursorPosition = persistence.CURSOR_MIDDLE c.CursorPosition = persistence.CURSOR_MIDDLE
c.CursorValue = 1644107102 c.CursorValue = 1644107102
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Retweets, 1) assert.Len(feed.Retweets, 1)
@ -103,7 +103,7 @@ func TestBuildUserFeedEnd(t *testing.T) {
c.PageSize = 2 c.PageSize = 2
c.CursorPosition = persistence.CURSOR_MIDDLE c.CursorPosition = persistence.CURSOR_MIDDLE
c.CursorValue = 1 // Won't be anything c.CursorValue = 1 // Won't be anything
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Retweets, 0) assert.Len(feed.Retweets, 0)
@ -114,6 +114,24 @@ func TestBuildUserFeedEnd(t *testing.T) {
assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END) assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END)
} }
func TestUserFeedHasLikesInfo(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Fetch @Peter_Nimitz user feed while logged in as @MysteryGrove
c := persistence.NewUserFeedCursor(UserHandle("Peter_Nimitz"))
feed, err := profile.NextPage(c, UserID(1178839081222115328))
require.NoError(err)
// Should have "liked" 1 tweet
for _, t := range feed.Tweets {
assert.Equal(t.IsLikedByCurrentUser, t.ID == TweetID(1413646595493568516))
}
}
func TestUserFeedWithTombstone(t *testing.T) { func TestUserFeedWithTombstone(t *testing.T) {
require := require.New(t) require := require.New(t)
assert := assert.New(t) assert := assert.New(t)
@ -122,7 +140,7 @@ func TestUserFeedWithTombstone(t *testing.T) {
require.NoError(err) require.NoError(err)
c := persistence.NewUserFeedCursor(UserHandle("Heminator")) c := persistence.NewUserFeedCursor(UserHandle("Heminator"))
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
tombstone_tweet := feed.Tweets[TweetID(31)] tombstone_tweet := feed.Tweets[TweetID(31)]
assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author") assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author")
@ -135,7 +153,7 @@ func TestTweetDetailWithReplies(t *testing.T) {
profile, err := persistence.LoadProfile("../../sample_data/profile") profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err) require.NoError(err)
tweet_detail, err := profile.GetTweetDetail(TweetID(1413646595493568516)) tweet_detail, err := profile.GetTweetDetail(TweetID(1413646595493568516), UserID(1178839081222115328))
require.NoError(err) require.NoError(err)
assert.Len(tweet_detail.Retweets, 0) assert.Len(tweet_detail.Retweets, 0)
@ -151,8 +169,9 @@ func TestTweetDetailWithReplies(t *testing.T) {
1413772782358433792, 1413772782358433792,
1413773185296650241, 1413773185296650241,
} { } {
_, is_ok := tweet_detail.Tweets[id] t, is_ok := tweet_detail.Tweets[id]
assert.True(is_ok) assert.True(is_ok)
assert.Equal(t.IsLikedByCurrentUser, id == 1413646595493568516)
} }
assert.Len(tweet_detail.Users, 4) assert.Len(tweet_detail.Users, 4)
@ -189,7 +208,7 @@ func TestTweetDetailWithParents(t *testing.T) {
profile, err := persistence.LoadProfile("../../sample_data/profile") profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err) require.NoError(err)
tweet_detail, err := profile.GetTweetDetail(TweetID(1413773185296650241)) tweet_detail, err := profile.GetTweetDetail(TweetID(1413773185296650241), UserID(1178839081222115328))
require.NoError(err) require.NoError(err)
assert.Len(tweet_detail.Retweets, 0) assert.Len(tweet_detail.Retweets, 0)
@ -201,8 +220,9 @@ func TestTweetDetailWithParents(t *testing.T) {
1413772782358433792, 1413772782358433792,
1413773185296650241, 1413773185296650241,
} { } {
_, is_ok := tweet_detail.Tweets[id] t, is_ok := tweet_detail.Tweets[id]
assert.True(is_ok) assert.True(is_ok)
assert.Equal(t.IsLikedByCurrentUser, id == 1413646595493568516)
} }
assert.Len(tweet_detail.Users, 2) assert.Len(tweet_detail.Users, 2)

13
pkg/persistence/compound_ssf_queries.go
View File

@ -261,7 +261,7 @@ func (c *Cursor) apply_token(token string) error {
return nil return nil
} }
func (p Profile) NextPage(c Cursor) (Feed, error) { func (p Profile) NextPage(c Cursor, current_user_id scraper.UserID) (Feed, error) {
where_clauses := []string{} where_clauses := []string{}
bind_values := []interface{}{} bind_values := []interface{}{}
@ -273,7 +273,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
// From, to, by, and RT'd by user handles // From, to, by, and RT'd by user handles
if c.FromUserHandle != "" { if c.FromUserHandle != "" {
where_clauses = append(where_clauses, "user_id = (select id from users where handle like ?)") where_clauses = append(where_clauses, "tweets.user_id = (select id from users where handle like ?)")
bind_values = append(bind_values, c.FromUserHandle) bind_values = append(bind_values, c.FromUserHandle)
} }
for _, to_user := range c.ToUserHandles { for _, to_user := range c.ToUserHandles {
@ -362,9 +362,10 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
q := `select * from ( q := `select * from (
select ` + TWEETS_ALL_SQL_FIELDS + `, select ` + TWEETS_ALL_SQL_FIELDS + `,
0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at, 0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at,
posted_at chrono, user_id by_user_id posted_at chrono, tweets.user_id by_user_id
from tweets from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
left join likes on tweets.id = likes.tweet_id and likes.user_id = ?
` + where_clause + ` ` + c.SortOrder.OrderByClause() + ` limit ? ` + where_clause + ` ` + c.SortOrder.OrderByClause() + ` limit ?
) )
@ -372,16 +373,18 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
select * from ( select * from (
select ` + TWEETS_ALL_SQL_FIELDS + `, select ` + TWEETS_ALL_SQL_FIELDS + `,
tweet_id, retweet_id, retweeted_by, retweeted_at, retweets.tweet_id, retweet_id, retweeted_by, retweeted_at,
retweeted_at chrono, retweeted_by by_user_id retweeted_at chrono, retweeted_by by_user_id
from retweets from retweets
left join tweets on retweets.tweet_id = tweets.id left join tweets on retweets.tweet_id = tweets.id
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
left join likes on tweets.id = likes.tweet_id and likes.user_id = ?
` + where_clause + ` ` + where_clause + `
` + c.SortOrder.OrderByClause() + ` ` + c.SortOrder.OrderByClause() + `
limit ? limit ?
) ` + c.SortOrder.OrderByClause() + ` limit ?` ) ` + c.SortOrder.OrderByClause() + ` limit ?`
bind_values = append([]interface{}{current_user_id}, bind_values...)
bind_values = append(bind_values, c.PageSize) bind_values = append(bind_values, c.PageSize)
bind_values = append(bind_values, bind_values...) bind_values = append(bind_values, bind_values...)
bind_values = append(bind_values, c.PageSize) bind_values = append(bind_values, c.PageSize)
@ -406,7 +409,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID}) ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID})
} }
p.fill_content(&ret.TweetTrove) p.fill_content(&ret.TweetTrove, current_user_id)
ret.CursorBottom = c ret.CursorBottom = c

36
pkg/persistence/compound_ssf_queries_test.go
View File

@ -25,7 +25,7 @@ func TestCursorSearchByNewest(t *testing.T) {
c.Keywords = []string{"think"} c.Keywords = []string{"think"}
c.SortOrder = persistence.SORT_ORDER_NEWEST c.SortOrder = persistence.SORT_ORDER_NEWEST
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 3) assert.Len(feed.Items, 3)
@ -41,7 +41,7 @@ func TestCursorSearchByNewest(t *testing.T) {
assert.Equal(next_cursor.PageSize, c.PageSize) assert.Equal(next_cursor.PageSize, c.PageSize)
assert.Equal(next_cursor.CursorValue, 1629520619) assert.Equal(next_cursor.CursorValue, 1629520619)
feed, err = profile.NextPage(next_cursor) feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 2) assert.Len(feed.Items, 2)
@ -67,7 +67,7 @@ func TestCursorSearchWithRetweets(t *testing.T) {
c.FilterRetweets = persistence.REQUIRE c.FilterRetweets = persistence.REQUIRE
c.SortOrder = persistence.SORT_ORDER_OLDEST c.SortOrder = persistence.SORT_ORDER_OLDEST
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 3) assert.Len(feed.Items, 3)
@ -83,7 +83,7 @@ func TestCursorSearchWithRetweets(t *testing.T) {
assert.Equal(next_cursor.PageSize, c.PageSize) assert.Equal(next_cursor.PageSize, c.PageSize)
assert.Equal(next_cursor.CursorValue, 1644111031) assert.Equal(next_cursor.CursorValue, 1644111031)
feed, err = profile.NextPage(next_cursor) feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 0) assert.Len(feed.Items, 0)
@ -102,7 +102,7 @@ func TestTimeline(t *testing.T) {
c := persistence.NewTimelineCursor() c := persistence.NewTimelineCursor()
c.PageSize = 5 c.PageSize = 5
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 5) assert.Len(feed.Items, 5)
@ -121,7 +121,7 @@ func TestTimeline(t *testing.T) {
assert.Equal(next_cursor.CursorValue, 1635367140) assert.Equal(next_cursor.CursorValue, 1635367140)
next_cursor.CursorValue = 1631935323 // Scroll down a bit, kind of randomly next_cursor.CursorValue = 1631935323 // Scroll down a bit, kind of randomly
feed, err = profile.NextPage(next_cursor) feed, err = profile.NextPage(next_cursor, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 5) assert.Len(feed.Items, 5)
@ -143,20 +143,20 @@ func TestKeywordSearch(t *testing.T) {
// Multiple words without quotes // Multiple words without quotes
c.Keywords = []string{"who", "are"} c.Keywords = []string{"who", "are"}
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.True(len(feed.Items) > 1) assert.True(len(feed.Items) > 1)
// Add quotes // Add quotes
c.Keywords = []string{"who are"} c.Keywords = []string{"who are"}
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 1) assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700)) assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700))
// With gibberish (no matches) // With gibberish (no matches)
c.Keywords = []string{"fasdfjkafsldfjsff"} c.Keywords = []string{"fasdfjkafsldfjsff"}
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 0) assert.Len(feed.Items, 0)
} }
@ -171,7 +171,7 @@ func TestSearchReplyingToUser(t *testing.T) {
// Replying to a user // Replying to a user
c.ToUserHandles = []UserHandle{"spacex"} c.ToUserHandles = []UserHandle{"spacex"}
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 2) assert.Len(feed.Items, 2)
assert.Equal(feed.Items[0].TweetID, TweetID(1428951883058753537)) assert.Equal(feed.Items[0].TweetID, TweetID(1428951883058753537))
@ -179,7 +179,7 @@ func TestSearchReplyingToUser(t *testing.T) {
// Replying to two users // Replying to two users
c.ToUserHandles = []UserHandle{"spacex", "covfefeanon"} c.ToUserHandles = []UserHandle{"spacex", "covfefeanon"}
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 1) assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1428939163961790466)) assert.Equal(feed.Items[0].TweetID, TweetID(1428939163961790466))
@ -197,7 +197,7 @@ func TestSearchDateFilters(t *testing.T) {
// Since timestamp // Since timestamp
c.SinceTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC) c.SinceTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC)
c.FromUserHandle = UserHandle("cernovich") c.FromUserHandle = UserHandle("cernovich")
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 1) assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1453461248142495744)) assert.Equal(feed.Items[0].TweetID, TweetID(1453461248142495744))
@ -205,7 +205,7 @@ func TestSearchDateFilters(t *testing.T) {
// Until timestamp // Until timestamp
c.SinceTimestamp = TimestampFromUnix(0) c.SinceTimestamp = TimestampFromUnix(0)
c.UntilTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC) c.UntilTimestamp.Time = time.Date(2021, 10, 1, 0, 0, 0, 0, time.UTC)
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 3) assert.Len(feed.Items, 3)
assert.Equal(feed.Items[0].TweetID, TweetID(1439027915404939265)) assert.Equal(feed.Items[0].TweetID, TweetID(1439027915404939265))
@ -224,7 +224,7 @@ func TestSearchMediaFilters(t *testing.T) {
c := persistence.NewCursor() c := persistence.NewCursor()
c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.SortOrder = persistence.SORT_ORDER_MOST_LIKES
c.FilterLinks = persistence.REQUIRE c.FilterLinks = persistence.REQUIRE
feed, err := profile.NextPage(c) feed, err := profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 2) assert.Len(feed.Items, 2)
assert.Equal(feed.Items[0].TweetID, TweetID(1438642143170646017)) assert.Equal(feed.Items[0].TweetID, TweetID(1438642143170646017))
@ -234,7 +234,7 @@ func TestSearchMediaFilters(t *testing.T) {
c = persistence.NewCursor() c = persistence.NewCursor()
c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.SortOrder = persistence.SORT_ORDER_MOST_LIKES
c.FilterImages = persistence.REQUIRE c.FilterImages = persistence.REQUIRE
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 2) assert.Len(feed.Items, 2)
assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700)) assert.Equal(feed.Items[0].TweetID, TweetID(1261483383483293700))
@ -244,7 +244,7 @@ func TestSearchMediaFilters(t *testing.T) {
c = persistence.NewCursor() c = persistence.NewCursor()
c.SortOrder = persistence.SORT_ORDER_MOST_LIKES c.SortOrder = persistence.SORT_ORDER_MOST_LIKES
c.FilterVideos = persistence.REQUIRE c.FilterVideos = persistence.REQUIRE
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 2) assert.Len(feed.Items, 2)
assert.Equal(feed.Items[0].TweetID, TweetID(1426619468327882761)) assert.Equal(feed.Items[0].TweetID, TweetID(1426619468327882761))
@ -253,7 +253,7 @@ func TestSearchMediaFilters(t *testing.T) {
// Polls // Polls
c = persistence.NewCursor() c = persistence.NewCursor()
c.FilterPolls = persistence.REQUIRE c.FilterPolls = persistence.REQUIRE
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 1) assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1465534109573390348)) assert.Equal(feed.Items[0].TweetID, TweetID(1465534109573390348))
@ -261,7 +261,7 @@ func TestSearchMediaFilters(t *testing.T) {
// Spaces // Spaces
c = persistence.NewCursor() c = persistence.NewCursor()
c.FilterSpaces = persistence.REQUIRE c.FilterSpaces = persistence.REQUIRE
feed, err = profile.NextPage(c) feed, err = profile.NextPage(c, UserID(0))
require.NoError(err) require.NoError(err)
assert.Len(feed.Items, 1) assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1624833173514293249)) assert.Equal(feed.Items[0].TweetID, TweetID(1624833173514293249))

1
pkg/scraper/tweet.go
View File

@ -73,6 +73,7 @@ type Tweet struct {
TombstoneText string `db:"tombstone_text"` TombstoneText string `db:"tombstone_text"`
IsStub bool `db:"is_stub"` IsStub bool `db:"is_stub"`
IsLikedByCurrentUser bool `db:"is_liked_by_current_user"`
IsContentDownloaded bool `db:"is_content_downloaded"` IsContentDownloaded bool `db:"is_content_downloaded"`
IsConversationScraped bool `db:"is_conversation_scraped"` IsConversationScraped bool `db:"is_conversation_scraped"`
LastScrapedAt Timestamp `db:"last_scraped_at"` LastScrapedAt Timestamp `db:"last_scraped_at"`

1
sample_data/seed_data.sql
View File

@ -287,6 +287,7 @@ create table likes(rowid integer primary key,
foreign key(user_id) references users(id) foreign key(user_id) references users(id)
foreign key(tweet_id) references tweets(id) foreign key(tweet_id) references tweets(id)
); );
insert into likes values(1, 1, 1178839081222115328, 1413646595493568516);
create table fake_user_sequence(latest_fake_id integer not null); create table fake_user_sequence(latest_fake_id integer not null);
insert into fake_user_sequence values(0x4000000000000000); insert into fake_user_sequence values(0x4000000000000000);