From 99f6de9d45cc1cde5e1eaa3d42e17f30c9c5e296 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 12 Nov 2023 13:04:02 -0800 Subject: [PATCH] Add UI queries for DMs --- internal/webserver/server_test.go | 2 +- pkg/persistence/dm_queries.go | 160 +++++++++++++++++++++++++++++ pkg/persistence/dm_queries_test.go | 71 +++++++++++++ pkg/scraper/dm_chat_room.go | 7 +- sample_data/seed_data.sql | 27 ++++- 5 files changed, 263 insertions(+), 4 deletions(-) diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index fdcd47f..fbc0ce5 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -564,5 +564,5 @@ func TestLists(t *testing.T) { resp := do_request(httptest.NewRequest("GET", "/lists", nil)) root, err := html.Parse(resp.Body) assert.NoError(err) - assert.Len(cascadia.QueryAll(root, selector(".users-list-container .author-info")), 4) + assert.Len(cascadia.QueryAll(root, selector(".users-list-container .author-info")), 5) } diff --git a/pkg/persistence/dm_queries.go b/pkg/persistence/dm_queries.go index 5358092..6f65e58 100644 --- a/pkg/persistence/dm_queries.go +++ b/pkg/persistence/dm_queries.go @@ -2,6 +2,9 @@ package persistence import ( "fmt" + "strings" + + "github.com/jmoiron/sqlx" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -142,3 +145,160 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) { } return ret, nil } + +type DMChatView struct { + DMTrove + RoomIDs []DMChatRoomID + MessageIDs []DMMessageID +} + +func NewDMChatView() DMChatView { + return DMChatView{ + DMTrove: NewDMTrove(), + RoomIDs: []DMChatRoomID{}, + MessageIDs: []DMMessageID{}, + } +} + +func (p Profile) GetChatRoomsPreview(id UserID) DMChatView { + ret := NewDMChatView() + + var rooms []DMChatRoom + err := p.DB.Select(&rooms, ` + select id, type, last_messaged_at, is_nsfw + from chat_rooms + where exists (select 1 from chat_room_participants where chat_room_id = chat_rooms.id and user_id = ?) + order by last_messaged_at desc + `, id) + if err != nil { + panic(err) + } + for _, room := range rooms { + // Fetch the latest message + var msg DMMessage + q, args, err := sqlx.Named(` + select id, chat_room_id, sender_id, sent_at, request_id, text, in_reply_to_id + from chat_messages + where chat_room_id = :room_id + and sent_at = (select max(sent_at) from chat_messages where chat_room_id = :room_id) + `, struct { + ID DMChatRoomID `db:"room_id"` + }{ID: room.ID}) + if err != nil { + panic(err) + } + err = p.DB.Get(&msg, q, args...) + if err != nil { + panic(err) + } + + // Fetch the participants + // DUPE chat-room-participants-SQL + var participants []struct { + DMChatParticipant + User + } + err = p.DB.Select(&participants, ` + select chat_room_id, user_id, last_read_event_id, is_chat_settings_valid, is_notifications_disabled, + is_mention_notifications_disabled, is_read_only, is_trusted, is_muted, status, `+USERS_ALL_SQL_FIELDS+` + from chat_room_participants join users on chat_room_participants.user_id = users.id + where chat_room_id = ? + `, room.ID) + if err != nil { + panic(err) + } + room.Participants = make(map[UserID]DMChatParticipant) + for _, participant := range participants { + room.Participants[participant.User.ID] = participant.DMChatParticipant + ret.Users[participant.User.ID] = participant.User + } + + // Add everything to the Trove + room.LastMessageID = msg.ID + ret.Rooms[room.ID] = room + ret.Messages[msg.ID] = msg + ret.RoomIDs = append(ret.RoomIDs, room.ID) + } + return ret +} + +func (p Profile) GetChatRoomContents(id DMChatRoomID) DMChatView { + ret := NewDMChatView() + var room DMChatRoom + err := p.DB.Get(&room, ` + select id, type, last_messaged_at, is_nsfw + from chat_rooms + where id = ? + `, id) + if err != nil { + panic(err) + } + + // Fetch the participants + // DUPE chat-room-participants-SQL + var participants []struct { + DMChatParticipant + User + } + err = p.DB.Select(&participants, ` + select chat_room_id, user_id, last_read_event_id, is_chat_settings_valid, is_notifications_disabled, + is_mention_notifications_disabled, is_read_only, is_trusted, is_muted, status, `+USERS_ALL_SQL_FIELDS+` + from chat_room_participants join users on chat_room_participants.user_id = users.id + where chat_room_id = ? + `, room.ID) + if err != nil { + panic(err) + } + room.Participants = make(map[UserID]DMChatParticipant) + for _, participant := range participants { + room.Participants[participant.User.ID] = participant.DMChatParticipant + ret.Users[participant.User.ID] = participant.User + } + + // Fetch all messages + var msgs []DMMessage + err = p.DB.Select(&msgs, ` + select id, chat_room_id, sender_id, sent_at, request_id, text, in_reply_to_id + from chat_messages + where chat_room_id = :room_id + order by sent_at desc + limit 50 + `, room.ID) + if err != nil { + panic(err) + } + ret.MessageIDs = make([]DMMessageID, len(msgs)) + for i, msg := range msgs { + ret.MessageIDs[len(ret.MessageIDs)-i-1] = msg.ID + msg.Reactions = make(map[UserID]DMReaction) + ret.Messages[msg.ID] = msg + } + + // Set last message ID on chat room + room.LastMessageID = ret.MessageIDs[len(ret.MessageIDs)-1] + + // Put the room in the Trove + ret.Rooms[room.ID] = room + + // Fetch all reaccs + var reaccs []DMReaction + message_ids_copy := make([]interface{}, len(ret.MessageIDs)) + for i, id := range ret.MessageIDs { + message_ids_copy[i] = id + } + err = p.DB.Select(&reaccs, ` + select id, message_id, sender_id, sent_at, emoji + from chat_message_reactions + where message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?) + `, message_ids_copy...) + if err != nil { + panic(err) + } + for _, reacc := range reaccs { + msg := ret.Messages[reacc.DMMessageID] + msg.Reactions[reacc.SenderID] = reacc + ret.Messages[reacc.DMMessageID] = msg + } + + return ret +} diff --git a/pkg/persistence/dm_queries_test.go b/pkg/persistence/dm_queries_test.go index 51b2691..c7df087 100644 --- a/pkg/persistence/dm_queries_test.go +++ b/pkg/persistence/dm_queries_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -151,3 +152,73 @@ func TestAddReactionToChatMessage(t *testing.T) { t.Error(diff) } } + +func TestGetChatRoomsPreview(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + chat_view := profile.GetChatRoomsPreview(UserID(1458284524761075714)) + assert.Len(chat_view.Rooms, 1) + assert.Len(chat_view.RoomIDs, 1) + assert.Equal(chat_view.RoomIDs, []DMChatRoomID{"1458284524761075714-1488963321701171204"}) + + room, is_ok := chat_view.Rooms[chat_view.RoomIDs[0]] + require.True(is_ok) + assert.Equal(room.LastMessageID, DMMessageID(1665936253483614212)) + + msg, is_ok := chat_view.Messages[room.LastMessageID] + require.True(is_ok) + assert.Equal(msg.Text, "Check this out\nhttps://t.co/rHeWGgNIZ1") + + require.Len(room.Participants, 2) + for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} { + participant, is_ok := room.Participants[user_id] + require.True(is_ok) + assert.Equal(participant.IsChatSettingsValid, participant.UserID == 1488963321701171204) + _, is_ok = chat_view.Users[user_id] + require.True(is_ok) + } +} + +func TestGetChatRoomContents(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + room_id := DMChatRoomID("1458284524761075714-1488963321701171204") + chat_view := profile.GetChatRoomContents(room_id) + assert.Len(chat_view.Rooms, 1) + room, is_ok := chat_view.Rooms[room_id] + require.True(is_ok) + + // Participants + require.Len(room.Participants, 2) + for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} { + participant, is_ok := room.Participants[user_id] + require.True(is_ok) + assert.Equal(participant.IsChatSettingsValid, participant.UserID == 1488963321701171204) + _, is_ok = chat_view.Users[user_id] + require.True(is_ok) + } + + // Messages + require.Equal(chat_view.MessageIDs, []DMMessageID{1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212}) + require.Len(chat_view.Messages, 4) + for _, msg_id := range chat_view.MessageIDs { + msg, is_ok := chat_view.Messages[msg_id] + assert.True(is_ok) + assert.Equal(msg.ID, msg_id) + } + + // Reactions + msg_with_reacc := chat_view.Messages[DMMessageID(1663623062195957773)] + require.Len(msg_with_reacc.Reactions, 1) + reacc, is_ok := msg_with_reacc.Reactions[UserID(1458284524761075714)] + require.True(is_ok) + assert.Equal(reacc.Emoji, "😂") +} diff --git a/pkg/scraper/dm_chat_room.go b/pkg/scraper/dm_chat_room.go index e137554..c1ac634 100644 --- a/pkg/scraper/dm_chat_room.go +++ b/pkg/scraper/dm_chat_room.go @@ -20,13 +20,16 @@ type DMChatParticipant struct { Status string `db:"status"` } +// A chat room. Stores a map of chat participants and a reference to the most recent message, +// for preview purposes. type DMChatRoom struct { ID DMChatRoomID `db:"id"` Type string `db:"type"` - LastMessagedAt Timestamp `db:"last_messaged_at"` + LastMessagedAt Timestamp `db:"last_messaged_at"` // Used for ordering the chats in the UI IsNSFW bool `db:"is_nsfw"` - Participants map[UserID]DMChatParticipant + LastMessageID DMMessageID `db:"last_message_id"` // Not stored, but used to generate preview + Participants map[UserID]DMChatParticipant } func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom { diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index 320007f..4ff179b 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -52,7 +52,8 @@ INSERT INTO users VALUES (97706,1159179478582603776,'Evelyn Kokemoor','EKokemoor','mars/wisconsin ⚧️⚢ ~macrep-racdec',256,139,'','',1565204898,0,0,0,'https://pbs.twimg.com/profile_images/1643762712868970497/r9JyQjKg.jpg','EKokemoor_profile_r9JyQjKg.jpg','https://pbs.twimg.com/profile_banners/1159179478582603776/1626219975','EKokemoor_banner_1626219975.jpg',1540465706139090944,0,0,0,0), (160242,534463724,'iko','ilyakooo0',replace('Code poet.\n~racfer-hattes','\n',char(10)),473,173,'','http://iko.soy',1332519666,0,0,0,'https://pbs.twimg.com/profile_images/1671427114438909952/8v8raTeb.jpg','ilyakooo0_profile_8v8raTeb.jpg','','',0,0,0,0,0), (169994,1689006330235760640,'sol🏴‍☠️','sol_plunder','',165,134,'','',1691525490,0,0,0,'https://pbs.twimg.com/profile_images/1689006644905033728/T1uO4Jvt.jpg','sol_plunder_profile_T1uO4Jvt.jpg','','',1704554384930058537,0,0,0,0), - (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0); + (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0), + (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one. Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0); create table tombstone_types (rowid integer primary key, short_name text not null unique, @@ -337,6 +338,9 @@ create table chat_rooms (rowid integer primary key, last_messaged_at integer not null, is_nsfw boolean not null ); +INSERT INTO chat_rooms VALUES + (1,'1458284524761075714-1488963321701171204','ONE_TO_ONE',1686025129132,0), + (2,'1488963321701171204-1178839081222115328','ONE_TO_ONE',1686025129144,0); create table chat_room_participants(rowid integer primary key, chat_room_id text not null, @@ -351,6 +355,11 @@ create table chat_room_participants(rowid integer primary key, status text not null, unique(chat_room_id, user_id) ); +INSERT INTO chat_room_participants VALUES + (1,'1458284524761075714-1488963321701171204',1458284524761075714,1665936253483614212,0,0,0,0,0,0,''), + (2,'1458284524761075714-1488963321701171204',1488963321701171204,1665936253483614212,1,0,0,0,1,0,'AT_END'), + (3,'1488963321701171204-1178839081222115328',1488963321701171204,1686075343331,1,0,0,0,1,0,'AT_END'), + (4,'1488963321701171204-1178839081222115328',1178839081222115328,1686075343331,0,0,0,0,0,0,''); create table chat_messages (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), @@ -363,6 +372,18 @@ create table chat_messages (rowid integer primary key, foreign key(chat_room_id) references chat_rooms(id) foreign key(sender_id) references users(id) ); +INSERT INTO chat_messages VALUES + (1,1663623062195957773,'1458284524761075714-1488963321701171204',1488963321701171204,1685473621419,'',0,'Yes helo'), + (2,1663623203644751885,'1458284524761075714-1488963321701171204',1458284524761075714,1685473655064,'',0,'Yeah i know who you are lol'), + (3,1665922180176044037,'1458284524761075714-1488963321701171204',1458284524761075714,1686021773787,'',1663623062195957773,'Yes?'), + (4,1665936253483614212,'1458284524761075714-1488963321701171204',1458284524761075714,1686025129132,'',0,replace('Check this out\nhttps://t.co/rHeWGgNIZ1','\n',char(10))), + (5,1665936253483614213,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129140,'',0,'bruh1'), + (6,1665936253483614214,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129141,'',0,'bruh2'), + (7,1665936253483614215,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129142,'',1665936253483614214,'replying to bruh2'), + (8,1665936253483614216,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129143,'',0,'This conversation is totally fake lol'), + (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly'); + + create table chat_message_reactions (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), @@ -373,6 +394,10 @@ create table chat_message_reactions (rowid integer primary key, foreign key(message_id) references chat_messages(id) foreign key(sender_id) references users(id) ); +INSERT INTO chat_message_reactions VALUES + (1,1665914315742781440,1663623062195957773,1458284524761075714,1686019898732,'😂'), + (2,1665936253487546456,1665936253483614216,1488963321701171204,1686063453455,'🤔'), + (3,1665936253834578774,1665936253483614216,1178839081222115328,1686075343331,'🤔'); create table fake_user_sequence(latest_fake_id integer not null);