Add DM API response structures and parsing of DM objects

pkg/scraper/api_types_dms.go

@@ -0,0 +1,57 @@
+package scraper
+
+type APIDMReaction struct {
+	ID       int    `json:"id,string"`
+	Time     int    `json:"time,string"`
+	SenderID int    `json:"sender_id,string"`
+	Emoji    string `json:"emoji_reaction"`
+}
+
+type APIDMMessage struct {
+	ID             int    `json:"id,string"`
+	Time           int    `json:"time,string"`
+	ConversationID string `json:"conversation_id"`
+	MessageData    struct {
+		ID        int    `json:"id,string"`
+		Time      int    `json:"time,string"`
+		SenderID  int    `json:"sender_id,string"`
+		Text      string `json:"text"`
+		ReplyData struct {
+			ID int `json:"id,string"`
+		} `json:"reply_data"`
+		Attachment struct {
+			Tweet APITweet `json:"tweet"`
+		} `json:"attachment"`
+	} `json:"message_data"`
+	MessageReactions []APIDMReaction `json:"message_reactions"`
+}
+
+type APIDMConversation struct {
+	ConversationID string `json:"conversation_id"`
+	Type           string `json:"type"`
+	SortTimestamp  int    `json:"sort_timestamp,string"`
+	Participants   []struct {
+		UserID          int `json:"user_id,string"`
+		LastReadEventID int `json:"last_read_event_id,string"`
+	}
+	NSFW                  bool   `json:"nsfw"`
+	NotificationsDisabled bool   `json:"notifications_disabled"`
+	ReadOnly              bool   `json:"read_only"`
+	Trusted               bool   `json:"trusted"`
+	Muted                 bool   `json:"muted"`
+	Status                string `json:"status"`
+}
+
+type APIInbox struct {
+	LastSeenEventID int    `json:"last_seen_event_id,string"`
+	Cursor          string `json:"cursor"`
+	Entries         []struct {
+		Message APIDMMessage `json:"message"`
+	} `json:"entries"`
+	Users         map[string]APIUser           `json:"users"`
+	Conversations map[string]APIDMConversation `json:"conversations"`
+}
+
+type APIDMResponse struct {
+	InboxInitialState APIInbox `json:"inbox_initial_state"`
+}

pkg/scraper/api_types_dms_test.go

@@ -0,0 +1,91 @@
+package scraper_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	. "offline_twitter/scraper"
+)
+
+func TestParseAPIDMMessage(t *testing.T) {
+	assert := assert.New(t)
+	data, err := os.ReadFile("test_responses/dms/dm_message.json")
+	if err != nil {
+		panic(err)
+	}
+	var api_message APIDMMessage
+	err = json.Unmarshal(data, &api_message)
+	require.NoError(t, err)
+
+	message := ParseAPIDMMessage(api_message)
+	assert.Equal(message.ID, DMMessageID(1663623203644751885))
+	assert.Equal(message.SentAt, TimestampFromUnix(1685473655064))
+	assert.Equal(message.DMChatRoomID, DMChatRoomID("1458284524761075714-1488963321701171204"))
+	assert.Equal(message.SenderID, UserID(1458284524761075714))
+	assert.Equal(message.Text, "Yeah i know who you are lol")
+	assert.Equal(message.InReplyToID, DMMessageID(0))
+	assert.Len(message.Reactions, 0)
+}
+
+func TestParseAPIDMMessageWithReaction(t *testing.T) {
+	assert := assert.New(t)
+	data, err := os.ReadFile("test_responses/dms/dm_message_with_reacc.json")
+	if err != nil {
+		panic(err)
+	}
+	var api_message APIDMMessage
+	err = json.Unmarshal(data, &api_message)
+	require.NoError(t, err)
+
+	message := ParseAPIDMMessage(api_message)
+	assert.Equal(message.ID, DMMessageID(1663623062195957773))
+	require.Len(t, message.Reactions, 1)
+
+	reacc := message.Reactions[0]
+	assert.Equal(reacc.ID, DMMessageID(1665914315742781440))
+	assert.Equal(reacc.SentAt, TimestampFromUnix(1686019898732))
+	assert.Equal(reacc.DMMessageID, DMMessageID(1663623062195957773))
+	assert.Equal(reacc.SenderID, UserID(1458284524761075714))
+	assert.Equal(reacc.Emoji, "😂")
+}
+
+func TestParseAPIDMConversation(t *testing.T) {
+	assert := assert.New(t)
+	data, err := os.ReadFile("test_responses/dms/dm_chat_room.json")
+	if err != nil {
+		panic(err)
+	}
+	var api_room APIDMConversation
+	err = json.Unmarshal(data, &api_room)
+	require.NoError(t, err)
+
+	// Simulate one of the participants being logged in
+	InitApi(API{UserID: 1458284524761075714})
+
+	chat_room := ParseAPIDMChatRoom(api_room)
+	assert.Equal(DMChatRoomID("1458284524761075714-1488963321701171204"), chat_room.ID)
+	assert.Equal("ONE_TO_ONE", chat_room.Type)
+	assert.Equal(TimestampFromUnix(1686025129086), chat_room.LastMessagedAt)
+	assert.False(chat_room.IsNSFW)
+
+	assert.Len(chat_room.Participants, 2)
+
+	p1 := chat_room.Participants[0]
+	assert.Equal(UserID(1458284524761075714), p1.UserID)
+	assert.Equal(DMMessageID(1665936253483614212), p1.LastReadEventID)
+	assert.True(p1.IsChatSettingsValid)
+	assert.False(p1.IsNotificationsDisabled)
+	assert.False(p1.IsReadOnly)
+	assert.True(p1.IsTrusted)
+	assert.False(p1.IsMuted)
+	assert.Equal("AT_END", p1.Status)
+
+	p2 := chat_room.Participants[1]
+	assert.Equal(UserID(1488963321701171204), p2.UserID)
+	assert.Equal(DMMessageID(1663623062195957773), p2.LastReadEventID)
+	assert.False(p2.IsChatSettingsValid)
+}

pkg/scraper/dm_chat_room.go

@@ -0,0 +1,56 @@
+package scraper
+
+import "fmt"
+
+type DMChatRoomID string
+
+type DMChatParticipant struct {
+	UserID          UserID
+	DMChatRoomID    DMChatRoomID
+	LastReadEventID DMMessageID
+
+	IsChatSettingsValid     bool
+	IsNotificationsDisabled bool
+	IsReadOnly              bool
+	IsTrusted               bool
+	IsMuted                 bool
+	Status                  string
+}
+
+type DMChatRoom struct {
+	ID             DMChatRoomID
+	Type           string
+	LastMessagedAt Timestamp
+	IsNSFW         bool
+
+	Participants []DMChatParticipant
+}
+
+func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom {
+	fmt.Printf("%#v\n", api_room)
+	ret := DMChatRoom{}
+	ret.ID = DMChatRoomID(api_room.ConversationID)
+	ret.Type = api_room.Type
+	ret.LastMessagedAt = TimestampFromUnix(int64(api_room.SortTimestamp))
+	ret.IsNSFW = api_room.NSFW
+
+	ret.Participants = []DMChatParticipant{}
+	for _, api_participant := range api_room.Participants {
+		participant := DMChatParticipant{}
+		participant.UserID = UserID(api_participant.UserID)
+		participant.DMChatRoomID = ret.ID
+		participant.LastReadEventID = DMMessageID(api_participant.LastReadEventID)
+
+		// Process chat settings if this is the logged-in user
+		if participant.UserID == the_api.UserID {
+			participant.IsNotificationsDisabled = api_room.NotificationsDisabled
+			participant.IsReadOnly = api_room.ReadOnly
+			participant.IsTrusted = api_room.Trusted
+			participant.IsMuted = api_room.Muted
+			participant.Status = api_room.Status
+			participant.IsChatSettingsValid = true
+		}
+		ret.Participants = append(ret.Participants, participant)
+	}
+	return ret
+}

pkg/scraper/dm_message.go

@@ -0,0 +1,33 @@
+package scraper
+
+type DMMessageID int
+
+type DMMessage struct {
+	ID           DMMessageID `db:"id"`
+	DMChatRoomID DMChatRoomID
+	SenderID     UserID
+	SentAt       Timestamp
+	RequestID    string
+	Text         string
+	InReplyToID  DMMessageID
+	Reactions    []DMReaction
+}
+
+func ParseAPIDMMessage(message APIDMMessage) DMMessage {
+	ret := DMMessage{}
+	ret.ID = DMMessageID(message.ID)
+	ret.SentAt = TimestampFromUnix(int64(message.Time))
+	ret.DMChatRoomID = DMChatRoomID(message.ConversationID)
+	ret.SenderID = UserID(message.MessageData.SenderID)
+	ret.Text = message.MessageData.Text
+
+	ret.InReplyToID = DMMessageID(message.MessageData.ReplyData.ID) // Will be "0" if not a reply
+
+	ret.Reactions = []DMReaction{}
+	for _, api_reacc := range message.MessageReactions {
+		reacc := ParseAPIDMReaction(api_reacc)
+		reacc.DMMessageID = ret.ID
+		ret.Reactions = append(ret.Reactions, reacc)
+	}
+	return ret
+}

pkg/scraper/dm_reaction.go

@@ -1,12 +1,18 @@
 package scraper
 
-type DMReactionID int
-
 type DMReaction struct {
-	ID             DMReactionID `db:"id"`
+	ID          DMMessageID `db:"id"`
-	Time           int
+	DMMessageID DMMessageID
-	ConversationID ConversationID
+	SenderID    UserID
-	MessageID      DMID
+	SentAt      Timestamp
-	ReactionKey    string
+	Emoji       string
-	SenderID       UserID
+}
+
+func ParseAPIDMReaction(reacc APIDMReaction) DMReaction {
+	ret := DMReaction{}
+	ret.ID = DMMessageID(reacc.ID)
+	ret.SenderID = UserID(reacc.SenderID)
+	ret.SentAt = TimestampFromUnix(int64(reacc.Time))
+	ret.Emoji = reacc.Emoji
+	return ret
 }