Add UI queries for DMs

internal/webserver/server_test.go

@@ -564,5 +564,5 @@ func TestLists(t *testing.T) {
 	resp := do_request(httptest.NewRequest("GET", "/lists", nil))
 	root, err := html.Parse(resp.Body)
 	assert.NoError(err)
-	assert.Len(cascadia.QueryAll(root, selector(".users-list-container .author-info")), 4)
+	assert.Len(cascadia.QueryAll(root, selector(".users-list-container .author-info")), 5)
 }

pkg/persistence/dm_queries.go

@@ -2,6 +2,9 @@ package persistence
 
 import (
 	"fmt"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
 
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
 )
@@ -142,3 +145,160 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) {
 	}
 	return ret, nil
 }
+
+type DMChatView struct {
+	DMTrove
+	RoomIDs    []DMChatRoomID
+	MessageIDs []DMMessageID
+}
+
+func NewDMChatView() DMChatView {
+	return DMChatView{
+		DMTrove:    NewDMTrove(),
+		RoomIDs:    []DMChatRoomID{},
+		MessageIDs: []DMMessageID{},
+	}
+}
+
+func (p Profile) GetChatRoomsPreview(id UserID) DMChatView {
+	ret := NewDMChatView()
+
+	var rooms []DMChatRoom
+	err := p.DB.Select(&rooms, `
+		select id, type, last_messaged_at, is_nsfw
+		  from chat_rooms
+		 where exists (select 1 from chat_room_participants where chat_room_id = chat_rooms.id and user_id = ?)
+		 order by last_messaged_at desc
+	`, id)
+	if err != nil {
+		panic(err)
+	}
+	for _, room := range rooms {
+		// Fetch the latest message
+		var msg DMMessage
+		q, args, err := sqlx.Named(`
+			select id, chat_room_id, sender_id, sent_at, request_id, text, in_reply_to_id
+		      from chat_messages
+		     where chat_room_id = :room_id
+		       and sent_at = (select max(sent_at) from chat_messages where chat_room_id = :room_id)
+		`, struct {
+			ID DMChatRoomID `db:"room_id"`
+		}{ID: room.ID})
+		if err != nil {
+			panic(err)
+		}
+		err = p.DB.Get(&msg, q, args...)
+		if err != nil {
+			panic(err)
+		}
+
+		// Fetch the participants
+		// DUPE chat-room-participants-SQL
+		var participants []struct {
+			DMChatParticipant
+			User
+		}
+		err = p.DB.Select(&participants, `
+			select chat_room_id, user_id, last_read_event_id, is_chat_settings_valid, is_notifications_disabled,
+			       is_mention_notifications_disabled, is_read_only, is_trusted, is_muted, status, `+USERS_ALL_SQL_FIELDS+`
+		      from chat_room_participants join users on chat_room_participants.user_id = users.id
+		     where chat_room_id = ?
+		`, room.ID)
+		if err != nil {
+			panic(err)
+		}
+		room.Participants = make(map[UserID]DMChatParticipant)
+		for _, participant := range participants {
+			room.Participants[participant.User.ID] = participant.DMChatParticipant
+			ret.Users[participant.User.ID] = participant.User
+		}
+
+		// Add everything to the Trove
+		room.LastMessageID = msg.ID
+		ret.Rooms[room.ID] = room
+		ret.Messages[msg.ID] = msg
+		ret.RoomIDs = append(ret.RoomIDs, room.ID)
+	}
+	return ret
+}
+
+func (p Profile) GetChatRoomContents(id DMChatRoomID) DMChatView {
+	ret := NewDMChatView()
+	var room DMChatRoom
+	err := p.DB.Get(&room, `
+		select id, type, last_messaged_at, is_nsfw
+	      from chat_rooms
+	     where id = ?
+	`, id)
+	if err != nil {
+		panic(err)
+	}
+
+	// Fetch the participants
+	// DUPE chat-room-participants-SQL
+	var participants []struct {
+		DMChatParticipant
+		User
+	}
+	err = p.DB.Select(&participants, `
+		select chat_room_id, user_id, last_read_event_id, is_chat_settings_valid, is_notifications_disabled,
+		       is_mention_notifications_disabled, is_read_only, is_trusted, is_muted, status, `+USERS_ALL_SQL_FIELDS+`
+	      from chat_room_participants join users on chat_room_participants.user_id = users.id
+	     where chat_room_id = ?
+	`, room.ID)
+	if err != nil {
+		panic(err)
+	}
+	room.Participants = make(map[UserID]DMChatParticipant)
+	for _, participant := range participants {
+		room.Participants[participant.User.ID] = participant.DMChatParticipant
+		ret.Users[participant.User.ID] = participant.User
+	}
+
+	// Fetch all messages
+	var msgs []DMMessage
+	err = p.DB.Select(&msgs, `
+		select id, chat_room_id, sender_id, sent_at, request_id, text, in_reply_to_id
+	      from chat_messages
+	     where chat_room_id = :room_id
+	     order by sent_at desc
+	     limit 50
+	`, room.ID)
+	if err != nil {
+		panic(err)
+	}
+	ret.MessageIDs = make([]DMMessageID, len(msgs))
+	for i, msg := range msgs {
+		ret.MessageIDs[len(ret.MessageIDs)-i-1] = msg.ID
+		msg.Reactions = make(map[UserID]DMReaction)
+		ret.Messages[msg.ID] = msg
+	}
+
+	// Set last message ID on chat room
+	room.LastMessageID = ret.MessageIDs[len(ret.MessageIDs)-1]
+
+	// Put the room in the Trove
+	ret.Rooms[room.ID] = room
+
+	// Fetch all reaccs
+	var reaccs []DMReaction
+	message_ids_copy := make([]interface{}, len(ret.MessageIDs))
+	for i, id := range ret.MessageIDs {
+		message_ids_copy[i] = id
+	}
+	err = p.DB.Select(&reaccs, `
+	    select id, message_id, sender_id, sent_at, emoji
+	      from chat_message_reactions
+		 where message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?)
+	`, message_ids_copy...)
+	if err != nil {
+		panic(err)
+	}
+	for _, reacc := range reaccs {
+		msg := ret.Messages[reacc.DMMessageID]
+		msg.Reactions[reacc.SenderID] = reacc
+		ret.Messages[reacc.DMMessageID] = msg
+	}
+
+	return ret
+}

pkg/persistence/dm_queries_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
 )
 
@@ -151,3 +152,73 @@ func TestAddReactionToChatMessage(t *testing.T) {
 		t.Error(diff)
 	}
 }
+
+func TestGetChatRoomsPreview(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	chat_view := profile.GetChatRoomsPreview(UserID(1458284524761075714))
+	assert.Len(chat_view.Rooms, 1)
+	assert.Len(chat_view.RoomIDs, 1)
+	assert.Equal(chat_view.RoomIDs, []DMChatRoomID{"1458284524761075714-1488963321701171204"})
+
+	room, is_ok := chat_view.Rooms[chat_view.RoomIDs[0]]
+	require.True(is_ok)
+	assert.Equal(room.LastMessageID, DMMessageID(1665936253483614212))
+
+	msg, is_ok := chat_view.Messages[room.LastMessageID]
+	require.True(is_ok)
+	assert.Equal(msg.Text, "Check this out\nhttps://t.co/rHeWGgNIZ1")
+
+	require.Len(room.Participants, 2)
+	for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} {
+		participant, is_ok := room.Participants[user_id]
+		require.True(is_ok)
+		assert.Equal(participant.IsChatSettingsValid, participant.UserID == 1488963321701171204)
+		_, is_ok = chat_view.Users[user_id]
+		require.True(is_ok)
+	}
+}
+
+func TestGetChatRoomContents(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	room_id := DMChatRoomID("1458284524761075714-1488963321701171204")
+	chat_view := profile.GetChatRoomContents(room_id)
+	assert.Len(chat_view.Rooms, 1)
+	room, is_ok := chat_view.Rooms[room_id]
+	require.True(is_ok)
+
+	// Participants
+	require.Len(room.Participants, 2)
+	for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} {
+		participant, is_ok := room.Participants[user_id]
+		require.True(is_ok)
+		assert.Equal(participant.IsChatSettingsValid, participant.UserID == 1488963321701171204)
+		_, is_ok = chat_view.Users[user_id]
+		require.True(is_ok)
+	}
+
+	// Messages
+	require.Equal(chat_view.MessageIDs, []DMMessageID{1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212})
+	require.Len(chat_view.Messages, 4)
+	for _, msg_id := range chat_view.MessageIDs {
+		msg, is_ok := chat_view.Messages[msg_id]
+		assert.True(is_ok)
+		assert.Equal(msg.ID, msg_id)
+	}
+
+	// Reactions
+	msg_with_reacc := chat_view.Messages[DMMessageID(1663623062195957773)]
+	require.Len(msg_with_reacc.Reactions, 1)
+	reacc, is_ok := msg_with_reacc.Reactions[UserID(1458284524761075714)]
+	require.True(is_ok)
+	assert.Equal(reacc.Emoji, "😂")
+}

pkg/scraper/dm_chat_room.go

@@ -20,13 +20,16 @@ type DMChatParticipant struct {
 	Status                         string `db:"status"`
 }
 
+// A chat room. Stores a map of chat participants and a reference to the most recent message,
+// for preview purposes.
 type DMChatRoom struct {
 	ID             DMChatRoomID `db:"id"`
 	Type           string       `db:"type"`
-	LastMessagedAt Timestamp    `db:"last_messaged_at"`
+	LastMessagedAt Timestamp    `db:"last_messaged_at"` // Used for ordering the chats in the UI
 	IsNSFW         bool         `db:"is_nsfw"`
 
-	Participants map[UserID]DMChatParticipant
+	LastMessageID DMMessageID `db:"last_message_id"` // Not stored, but used to generate preview
+	Participants  map[UserID]DMChatParticipant
 }
 
 func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom {

sample_data/seed_data.sql

@@ -52,7 +52,8 @@ INSERT INTO users VALUES
     (97706,1159179478582603776,'Evelyn Kokemoor','EKokemoor','mars/wisconsin ⚧️⚢ ~macrep-racdec',256,139,'','',1565204898,0,0,0,'https://pbs.twimg.com/profile_images/1643762712868970497/r9JyQjKg.jpg','EKokemoor_profile_r9JyQjKg.jpg','https://pbs.twimg.com/profile_banners/1159179478582603776/1626219975','EKokemoor_banner_1626219975.jpg',1540465706139090944,0,0,0,0),
     (160242,534463724,'iko','ilyakooo0',replace('Code poet.\n~racfer-hattes','\n',char(10)),473,173,'','http://iko.soy',1332519666,0,0,0,'https://pbs.twimg.com/profile_images/1671427114438909952/8v8raTeb.jpg','ilyakooo0_profile_8v8raTeb.jpg','','',0,0,0,0,0),
     (169994,1689006330235760640,'sol🏴‍☠️','sol_plunder','',165,134,'','',1691525490,0,0,0,'https://pbs.twimg.com/profile_images/1689006644905033728/T1uO4Jvt.jpg','sol_plunder_profile_T1uO4Jvt.jpg','','',1704554384930058537,0,0,0,0),
-    (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0);
+    (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0),
+    (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one.  Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0);
 
 create table tombstone_types (rowid integer primary key,
     short_name text not null unique,
@@ -337,6 +338,9 @@ create table chat_rooms (rowid integer primary key,
     last_messaged_at integer not null,
     is_nsfw boolean not null
 );
+INSERT INTO chat_rooms VALUES
+    (1,'1458284524761075714-1488963321701171204','ONE_TO_ONE',1686025129132,0),
+    (2,'1488963321701171204-1178839081222115328','ONE_TO_ONE',1686025129144,0);
 
 create table chat_room_participants(rowid integer primary key,
     chat_room_id text not null,
@@ -351,6 +355,11 @@ create table chat_room_participants(rowid integer primary key,
     status text not null,
     unique(chat_room_id, user_id)
 );
+INSERT INTO chat_room_participants VALUES
+    (1,'1458284524761075714-1488963321701171204',1458284524761075714,1665936253483614212,0,0,0,0,0,0,''),
+    (2,'1458284524761075714-1488963321701171204',1488963321701171204,1665936253483614212,1,0,0,0,1,0,'AT_END'),
+    (3,'1488963321701171204-1178839081222115328',1488963321701171204,1686075343331,1,0,0,0,1,0,'AT_END'),
+    (4,'1488963321701171204-1178839081222115328',1178839081222115328,1686075343331,0,0,0,0,0,0,'');
 
 create table chat_messages (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
@@ -363,6 +372,18 @@ create table chat_messages (rowid integer primary key,
     foreign key(chat_room_id) references chat_rooms(id)
     foreign key(sender_id) references users(id)
 );
+INSERT INTO chat_messages VALUES
+    (1,1663623062195957773,'1458284524761075714-1488963321701171204',1488963321701171204,1685473621419,'',0,'Yes helo'),
+    (2,1663623203644751885,'1458284524761075714-1488963321701171204',1458284524761075714,1685473655064,'',0,'Yeah i know who you are lol'),
+    (3,1665922180176044037,'1458284524761075714-1488963321701171204',1458284524761075714,1686021773787,'',1663623062195957773,'Yes?'),
+    (4,1665936253483614212,'1458284524761075714-1488963321701171204',1458284524761075714,1686025129132,'',0,replace('Check this out\nhttps://t.co/rHeWGgNIZ1','\n',char(10))),
+    (5,1665936253483614213,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129140,'',0,'bruh1'),
+    (6,1665936253483614214,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129141,'',0,'bruh2'),
+    (7,1665936253483614215,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129142,'',1665936253483614214,'replying to bruh2'),
+    (8,1665936253483614216,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129143,'',0,'This conversation is totally fake lol'),
+    (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly');
+
+
 
 create table chat_message_reactions (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
@@ -373,6 +394,10 @@ create table chat_message_reactions (rowid integer primary key,
     foreign key(message_id) references chat_messages(id)
     foreign key(sender_id) references users(id)
 );
+INSERT INTO chat_message_reactions VALUES
+    (1,1665914315742781440,1663623062195957773,1458284524761075714,1686019898732,'😂'),
+    (2,1665936253487546456,1665936253483614216,1488963321701171204,1686063453455,'🤔'),
+    (3,1665936253834578774,1665936253483614216,1178839081222115328,1686075343331,'🤔');
 
 
 create table fake_user_sequence(latest_fake_id integer not null);