diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index e8df2dd..cbefe64 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -254,7 +254,7 @@ func TestTimeline(t *testing.T) { assert.Equal(title_node.FirstChild.Data, "Timeline | Offline Twitter") tweet_nodes := cascadia.QueryAll(root, selector(".timeline > .tweet")) - assert.Len(tweet_nodes, 19) + assert.Len(tweet_nodes, 20) } func TestTimelineWithCursor(t *testing.T) { diff --git a/pkg/persistence/compound_queries.go b/pkg/persistence/compound_queries.go index 342c24d..3a6d226 100644 --- a/pkg/persistence/compound_queries.go +++ b/pkg/persistence/compound_queries.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/jmoiron/sqlx" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -13,6 +14,7 @@ var ( ErrNotInDB = errors.New("not in database") ) +// TODO: make this a SQL view? const TWEETS_ALL_SQL_FIELDS = ` tweets.id id, tweets.user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, @@ -20,6 +22,11 @@ const TWEETS_ALL_SQL_FIELDS = ` case when likes.user_id is null then 0 else 1 end is_liked_by_current_user, is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at` +// Given a TweetTrove, fetch its: +// - quoted tweets +// - spaces +// - users +// - images, videos, urls, polls func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { if len(trove.Tweets) == 0 { // Empty trove, nothing to fetch @@ -50,6 +57,7 @@ func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { } } + // Fetch spaces space_ids := []interface{}{} for _, t := range trove.Tweets { if t.SpaceID != "" { @@ -79,6 +87,7 @@ func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { } } + // Assemble list of users fetched in previous operations in_clause := "" user_ids := []interface{}{} tweet_ids := []interface{}{} @@ -98,6 +107,16 @@ func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { user_ids = append(user_ids, p) } } + for _, n := range trove.Notifications { + // Primary user + if n.ActionUserID != UserID(0) { + user_ids = append(user_ids, n.ActionUserID) + } + // Other users, if there are any + for _, u_id := range n.UserIDs { + user_ids = append(user_ids, u_id) + } + } // Get all the users if len(user_ids) > 0 { // It could be a search with no results, end of feed, etc-- strings.Repeat will fail! @@ -120,7 +139,6 @@ func (p Profile) fill_content(trove *TweetTrove, current_user_id UserID) { var images []Image imgquery := ` select id, tweet_id, width, height, remote_url, local_filename, is_downloaded from images where tweet_id in (` + in_clause + `)` - // fmt.Printf("%s\n", imgquery) // TODO: SQL logger err := p.DB.Select(&images, imgquery, tweet_ids...) if err != nil { panic(err) @@ -357,8 +375,9 @@ func (p Profile) GetTweetDetail(id TweetID, current_user_id UserID) (TweetDetail // TODO: compound-query-structs type FeedItem struct { TweetID - RetweetID TweetID - QuoteNestingLevel int + RetweetID TweetID + NotificationID + QuoteNestingLevel int // Defines the current nesting level (not available remaining levels) } type Feed struct { Items []FeedItem @@ -383,3 +402,114 @@ func NewFeed() Feed { TweetTrove: NewTweetTrove(), } } + +func (p Profile) GetNotificationsForUser(u_id UserID, cursor int64) Feed { + // Get the notifications + var notifications []Notification + err := p.DB.Select(¬ifications, + `select id, type, sent_at, sort_index, user_id, ifnull(action_user_id, 0) action_user_id, + ifnull(action_tweet_id, 0) action_tweet_id, ifnull(action_retweet_id, 0) action_retweet_id, has_detail, last_scraped_at + from notifications + where sort_index < ? or ? + and user_id = ? + order by sort_index desc + `, cursor, cursor == 0, u_id) + if err != nil { + panic(err) + } + + // Get the user_ids list for each notification. Unlike tweet+retweet_ids, users are needed to render + // the notification properly. + for i := range notifications { + err = p.DB.Select(¬ifications[i].UserIDs, `select user_id from notification_users where notification_id = ?`, notifications[i].ID) + if err != nil { + panic(err) + } + } + + // Collect tweet and retweet IDs + retweet_ids := []TweetID{} + tweet_ids := []TweetID{} + for _, n := range notifications { + if n.ActionRetweetID != TweetID(0) { + retweet_ids = append(retweet_ids, n.ActionRetweetID) + } + if n.ActionTweetID != TweetID(0) { + tweet_ids = append(tweet_ids, n.ActionTweetID) + } + } + + // TODO: can this go in `fill_content`? + + // Get retweets if there are any + var retweets []Retweet + if len(retweet_ids) != 0 { + sql_str, vals, err := sqlx.In(` + select retweet_id, tweet_id, retweeted_by, retweeted_at + from retweets + where retweet_id in (?) + `, retweet_ids) + if err != nil { + panic(err) + } + err = p.DB.Select(&retweets, sql_str, vals...) + if err != nil { + panic(err) + } + + // Collect more tweet IDs, from retweets + for _, r := range retweets { + tweet_ids = append(tweet_ids, r.TweetID) + } + } + + // Get tweets, if there are any + var tweets []Tweet + if len(tweet_ids) != 0 { + sql_str, vals, err := sqlx.In(`select `+TWEETS_ALL_SQL_FIELDS+` + from tweets + left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid + left join likes on tweets.id = likes.tweet_id and likes.user_id = -1 + where id in (?)`, tweet_ids) + if err != nil { + panic(err) + } + err = p.DB.Select(&tweets, sql_str, vals...) + if err != nil { + panic(err) + } + } + + ret := NewFeed() + for _, t := range tweets { + ret.TweetTrove.Tweets[t.ID] = t + } + for _, r := range retweets { + ret.TweetTrove.Retweets[r.RetweetID] = r + } + for _, n := range notifications { + // Add to tweet trove + ret.TweetTrove.Notifications[n.ID] = n + + // Construct feed item + feed_item := FeedItem{ + NotificationID: n.ID, + RetweetID: n.ActionRetweetID, // might be 0 + TweetID: n.ActionTweetID, // might be 0 + } + r, is_ok := ret.TweetTrove.Retweets[n.ActionRetweetID] + if is_ok { + // If the action has a retweet, fill the FeedItem.TweetID from the retweet + feed_item.TweetID = r.TweetID + } + ret.Items = append(ret.Items, feed_item) + } + + // TODO: proper user id + p.fill_content(&ret.TweetTrove, UserID(0)) + + // TODO: + // ret.CursorBottom = ?? + + return ret +} diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index 89d9b50..55c1ab9 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -317,3 +317,39 @@ func TestTweetDetailWithThread(t *testing.T) { require.Len(tweet_detail.ReplyChains, 1) // Should not include the Thread replies assert.Equal(tweet_detail.ReplyChains[0][0], TweetID(1698792233619562866)) } + +func TestNotificationsFeed(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + feed := profile.GetNotificationsForUser(UserID(1488963321701171204), 12345678912345) + assert.Len(feed.TweetTrove.Notifications, 6) + assert.Len(feed.TweetTrove.Tweets, 3) + assert.Len(feed.TweetTrove.Retweets, 1) + assert.Len(feed.TweetTrove.Users, 6) + + // Check that Users were retrieved on the notification with detail + notif, is_ok := feed.TweetTrove.Notifications["FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4"] + assert.True(is_ok) + assert.Len(notif.UserIDs, 3) + // Ensure they're also in the TweetTrove + for _, u_id := range notif.UserIDs { + _, is_ok := feed.TweetTrove.Users[u_id] + assert.True(is_ok) + } + + assert.Len(feed.Items, 6) + assert.Equal(feed.Items[0].NotificationID, NotificationID("FDzeDIfVUAIAAAABiJONcqaBFAzeN-n-Luw")) + assert.Equal(feed.Items[0].RetweetID, TweetID(1490135787124232223)) + assert.Equal(feed.Items[1].NotificationID, NotificationID("FDzeDIfVUAIAAvsBiJONcqYgiLgXOolO9t0")) + assert.Equal(feed.Items[1].TweetID, TweetID(1826778617705115869)) + assert.Equal(feed.Items[2].NotificationID, NotificationID("FKncQJGVgAQAAAABSQ3bEaTgXL8VBxefepo")) + assert.Equal(feed.Items[2].TweetID, TweetID(1826778617705115868)) + assert.Equal(feed.Items[3].NotificationID, NotificationID("FKncQJGVgAQAAAABSQ3bEaTgXL_S11Ev36g")) + assert.Equal(feed.Items[4].NotificationID, NotificationID("FKncQJGVgAQAAAABSQ3bEaTgXL-G8wObqVY")) + assert.Equal(feed.Items[5].NotificationID, NotificationID("FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4")) + assert.Equal(feed.Items[5].TweetID, TweetID(1826778617705115868)) +} diff --git a/pkg/persistence/compound_ssf_queries_test.go b/pkg/persistence/compound_ssf_queries_test.go index a042d49..76ca0b9 100644 --- a/pkg/persistence/compound_ssf_queries_test.go +++ b/pkg/persistence/compound_ssf_queries_test.go @@ -100,18 +100,19 @@ func TestTimeline(t *testing.T) { require.NoError(err) c := persistence.NewTimelineCursor() - c.PageSize = 5 + c.PageSize = 6 feed, err := profile.NextPage(c, UserID(0)) require.NoError(err) - assert.Len(feed.Items, 5) + assert.Len(feed.Items, 6) assert.Len(feed.Retweets, 4) - assert.Equal(feed.Items[0].RetweetID, TweetID(1490135787144237058)) - assert.Equal(feed.Items[1].RetweetID, TweetID(1490135787124232222)) - assert.Equal(feed.Items[2].RetweetID, TweetID(1490119308692766723)) - assert.Equal(feed.Items[3].RetweetID, TweetID(1490100255987171332)) - assert.Equal(feed.Items[4].TweetID, TweetID(1453461248142495744)) + assert.Equal(feed.Items[0].TweetID, TweetID(1826778617705115868)) + assert.Equal(feed.Items[1].RetweetID, TweetID(1490135787144237058)) + assert.Equal(feed.Items[2].RetweetID, TweetID(1490135787124232223)) + assert.Equal(feed.Items[3].RetweetID, TweetID(1490119308692766723)) + assert.Equal(feed.Items[4].RetweetID, TweetID(1490100255987171332)) + assert.Equal(feed.Items[5].TweetID, TweetID(1453461248142495744)) next_cursor := feed.CursorBottom assert.Equal(next_cursor.CursorPosition, persistence.CURSOR_MIDDLE) @@ -121,6 +122,7 @@ func TestTimeline(t *testing.T) { assert.Equal(next_cursor.CursorValue, 1635367140000) next_cursor.CursorValue = 1631935323000 // Scroll down a bit, kind of randomly + next_cursor.PageSize = 5 feed, err = profile.NextPage(next_cursor, UserID(0)) require.NoError(err) diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index ba38b99..d0599ff 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -53,7 +53,9 @@ INSERT INTO users VALUES (160242,534463724,'iko','ilyakooo0',replace('Code poet.\n~racfer-hattes','\n',char(10)),473,173,'','http://iko.soy',1332519666000,0,0,0,'https://pbs.twimg.com/profile_images/1671427114438909952/8v8raTeb.jpg','ilyakooo0_profile_8v8raTeb.jpg','','',0,0,0,0,0), (169994,1689006330235760640,'sol🏴‍☠️','sol_plunder','',165,134,'','',1691525490000,0,0,0,'https://pbs.twimg.com/profile_images/1689006644905033728/T1uO4Jvt.jpg','sol_plunder_profile_T1uO4Jvt.jpg','','',1704554384930058537,0,0,0,0), (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116000,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0), - (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one. Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522000,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0); + (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one. Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522000,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0), + (175560,249206992,'ludwig','ludwigABAP','God’s chosen principal engineer. What is impossible for you is not impossible for me.',984,17966,'','https://ludwigabap.bearblog.dev/',1297180819000,0,0,0,'https://pbs.twimg.com/profile_images/1753215006697459712/n76_qnTj.jpg','ludwigABAP_profile_n76_qnTj.jpg','https://pbs.twimg.com/profile_banners/249206992/1706835247','ludwigABAP_banner_1706835247.jpg',0,0,0,0,0); + create table lists(rowid integer primary key, is_online boolean not null default 0, @@ -192,7 +194,9 @@ INSERT INTO tweets VALUES (1409940,1698797388914151523,1458284524761075714,replace('This looks quite neat, but "comptime" is cool because it was designed to do stuff like declaring arrays where the size is the result of a function call, e.g.\n\nvar my_array: [fibonacci(10)]u32;\n\n...yet being able to create DSLs just emerged from this very simple concept','\n',char(10)),1693859834000,2,0,1,0,1698792233619562866,0,'ilyakooo0','ilyakooo0','',NULL,NULL,0,1,0,0,0), (1409953,1698802806096846909,1689006330235760640,replace('Just poking around at some examples and explanation videos, It does seem very similar to Template Haskell, though maybe a bit more ergonomic.\n\nIs there something missing from this mental model?','\n',char(10)),1693861125000,3,0,1,0,1698797388914151523,0,'wispem_wantex,ilyakooo0','wispem_wantex,ilyakooo0','',NULL,NULL,0,1,0,0,0), (1411566,1698848086880133147,1458284524761075714,'I have basically no experience with one and literally no experience with the other, and additionally I''ve never even used Haskell. So unfortunately I''m not really in a position to say.',1693871921000,1,0,1,0,1698802806096846909,0,'sol_plunder,ilyakooo0','sol_plunder,ilyakooo0','',NULL,NULL, 0,1,0,0,0), - (1169437,1665509126737129472,1458284524761075714,replace('Btw, to the extent this has happened, it''s partly thanks to the Golden One (@TheGloriousLion) who invented #fizeekfriday and the "post physique" rejoinder. Everyone should follow him if they don''t already.\n\nSince I forgot last week, and since it''s topical, here''s a leg poast','\n',char(10)),1685923294000,7,0,0,0,1665505986184900611,0,'TheGloriousLion','','fizeekfriday',NULL,NULL,0,1,0,0,0); + (1169437,1665509126737129472,1458284524761075714,replace('Btw, to the extent this has happened, it''s partly thanks to the Golden One (@TheGloriousLion) who invented #fizeekfriday and the "post physique" rejoinder. Everyone should follow him if they don''t already.\n\nSince I forgot last week, and since it''s topical, here''s a leg poast','\n',char(10)),1685923294000,7,0,0,0,1665505986184900611,0,'TheGloriousLion','','fizeekfriday',NULL,NULL,0,1,0,0,0), + (2857438,1826778617705115868,1488963321701171204,'Conversations are trees, not sequences. They branch. They don''t flow in a perfectly linear way.',1724372937000,4,1,0,0,0,0,'','','',NULL,NULL,0,1,0,0,0), + (2857439,1826778617705115869,1178839081222115328,'Real tweet that is definitely real',1724372938000,4,1,0,0,1826778617705115868,0,'Offline_Twatter','Offline_Twatter','',NULL,NULL,0,1,0,0,0); CREATE TABLE retweets(rowid integer primary key, retweet_id integer not null unique, @@ -211,7 +215,7 @@ INSERT INTO retweets VALUES (52,1490135787144237058,1490120332484972549,358545917,1644111031000), (42,1490119308692766723,1490116725395927042,358545917,1644107102000), (59,1490100255987171332,1489944024278523906,358545917,1644102560000), - (1000,1490135787124232222,1343715029707796489,1304281147074064385,1644111021000); -- This is fake + (1000,1490135787124232223,1698762413393236329,1488963321701171204,1644111022000); -- This is fake create table urls (rowid integer primary key, tweet_id integer not null, @@ -526,12 +530,75 @@ insert into follows values (3, 1488963321701171204, 1240784920831762433); +create table notification_types (rowid integer primary key, + name text not null unique +); +insert into notification_types(rowid, name) values + (1, 'like'), + (2, 'retweet'), + (3, 'quote-tweet'), + (4, 'reply'), + (5, 'follow'), + (6, 'mention'), + (7, 'user is LIVE'), + (8, 'poll ended'), + (9, 'login'), + (10, 'community pinned post'), + (11, 'new recommended post'); +create table notifications (rowid integer primary key, + id text unique, + type integer not null, + sent_at integer not null, + sort_index integer not null, + user_id integer not null, -- user who received the notification + + action_user_id integer references users(id), -- user who triggered the notification + action_tweet_id integer references tweets(id), -- tweet associated with the notification + action_retweet_id integer references retweets(retweet_id), + + has_detail boolean not null default 0, + last_scraped_at not null default 0, + + foreign key(type) references notification_types(rowid) + foreign key(user_id) references users(id) +); +INSERT INTO notifications VALUES + (1,'FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4',1,1723494244885,1723494244885,1488963321701171204,249206992,1826778617705115868,NULL,1,1725067356270), + (2,'FKncQJGVgAQAAAABSQ3bEaTgXL-G8wObqVY',9,1724112169072,1724112169072,1488963321701171204,NULL,NULL,NULL,0,-62135596800000), + (3,'FKncQJGVgAQAAAABSQ3bEaTgXL_S11Ev36g',5,1722251072880,1724251072880,1488963321701171204,1032468021485293568,NULL,NULL,0,-62135596800000), + (4,'FKncQJGVgAQAAAABSQ3bEaTgXL8VBxefepo',2,1724372973735,1724372973735,1488963321701171204,1178839081222115328,1826778617705115868,NULL,0,-62135596800000), + (5,'FDzeDIfVUAIAAvsBiJONcqYgiLgXOolO9t0',6,-62135596800000,1725055975543,1488963321701171204,1178839081222115328,1826778617705115869,NULL,0,-62135596800000), + (6,'FDzeDIfVUAIAAAABiJONcqaBFAzeN-n-Luw',1,1724604756351,1726604756351,1488963321701171204,1178839081222115328,NULL,1490135787124232223,0,-62135596800000); + +create table notification_tweets (rowid integer primary key, + notification_id not null references notifications(id), + tweet_id not null references tweets(id), + unique(notification_id, tweet_id) +); + +create table notification_retweets (rowid integer primary key, + notification_id not null references notifications(id), + retweet_id not null references retweets(retweet_id), + unique(notification_id, retweet_id) +); + +create table notification_users (rowid integer primary key, + notification_id not null references notifications(id), + user_id not null references users(id), + unique(notification_id, user_id) +); +INSERT INTO notification_users VALUES + (1,'FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4',249206992), + (2,'FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4',1304281147074064385), + (3,'FKncQJGVgAQAAAABSQ3bEaTgXL8f40e77r4',1178839081222115328); + + create table fake_user_sequence(latest_fake_id integer not null); insert into fake_user_sequence values(0x4000000000000000); create table database_version(rowid integer primary key, version_number integer not null unique ); -insert into database_version(version_number) values (30); +insert into database_version(version_number) values (31); COMMIT;