From 616ff30a1233500ac80f997f9653187e99656d0b Mon Sep 17 00:00:00 2001 From: Alessio Date: Tue, 6 Jun 2023 22:54:23 -0300 Subject: [PATCH] Add DM API response structures and parsing of DM objects --- pkg/scraper/api_types_dms.go | 57 +++++++++++++++++++ pkg/scraper/api_types_dms_test.go | 91 +++++++++++++++++++++++++++++++ pkg/scraper/dm_chat_room.go | 56 +++++++++++++++++++ pkg/scraper/dm_message.go | 33 +++++++++++ pkg/scraper/dm_reaction.go | 22 +++++--- 5 files changed, 251 insertions(+), 8 deletions(-) create mode 100644 pkg/scraper/api_types_dms.go create mode 100644 pkg/scraper/api_types_dms_test.go create mode 100644 pkg/scraper/dm_chat_room.go create mode 100644 pkg/scraper/dm_message.go diff --git a/pkg/scraper/api_types_dms.go b/pkg/scraper/api_types_dms.go new file mode 100644 index 0000000..c9920ef --- /dev/null +++ b/pkg/scraper/api_types_dms.go @@ -0,0 +1,57 @@ +package scraper + +type APIDMReaction struct { + ID int `json:"id,string"` + Time int `json:"time,string"` + SenderID int `json:"sender_id,string"` + Emoji string `json:"emoji_reaction"` +} + +type APIDMMessage struct { + ID int `json:"id,string"` + Time int `json:"time,string"` + ConversationID string `json:"conversation_id"` + MessageData struct { + ID int `json:"id,string"` + Time int `json:"time,string"` + SenderID int `json:"sender_id,string"` + Text string `json:"text"` + ReplyData struct { + ID int `json:"id,string"` + } `json:"reply_data"` + Attachment struct { + Tweet APITweet `json:"tweet"` + } `json:"attachment"` + } `json:"message_data"` + MessageReactions []APIDMReaction `json:"message_reactions"` +} + +type APIDMConversation struct { + ConversationID string `json:"conversation_id"` + Type string `json:"type"` + SortTimestamp int `json:"sort_timestamp,string"` + Participants []struct { + UserID int `json:"user_id,string"` + LastReadEventID int `json:"last_read_event_id,string"` + } + NSFW bool `json:"nsfw"` + NotificationsDisabled bool `json:"notifications_disabled"` + ReadOnly bool `json:"read_only"` + Trusted bool `json:"trusted"` + Muted bool `json:"muted"` + Status string `json:"status"` +} + +type APIInbox struct { + LastSeenEventID int `json:"last_seen_event_id,string"` + Cursor string `json:"cursor"` + Entries []struct { + Message APIDMMessage `json:"message"` + } `json:"entries"` + Users map[string]APIUser `json:"users"` + Conversations map[string]APIDMConversation `json:"conversations"` +} + +type APIDMResponse struct { + InboxInitialState APIInbox `json:"inbox_initial_state"` +} diff --git a/pkg/scraper/api_types_dms_test.go b/pkg/scraper/api_types_dms_test.go new file mode 100644 index 0000000..5094bc3 --- /dev/null +++ b/pkg/scraper/api_types_dms_test.go @@ -0,0 +1,91 @@ +package scraper_test + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + . "offline_twitter/scraper" +) + +func TestParseAPIDMMessage(t *testing.T) { + assert := assert.New(t) + data, err := os.ReadFile("test_responses/dms/dm_message.json") + if err != nil { + panic(err) + } + var api_message APIDMMessage + err = json.Unmarshal(data, &api_message) + require.NoError(t, err) + + message := ParseAPIDMMessage(api_message) + assert.Equal(message.ID, DMMessageID(1663623203644751885)) + assert.Equal(message.SentAt, TimestampFromUnix(1685473655064)) + assert.Equal(message.DMChatRoomID, DMChatRoomID("1458284524761075714-1488963321701171204")) + assert.Equal(message.SenderID, UserID(1458284524761075714)) + assert.Equal(message.Text, "Yeah i know who you are lol") + assert.Equal(message.InReplyToID, DMMessageID(0)) + assert.Len(message.Reactions, 0) +} + +func TestParseAPIDMMessageWithReaction(t *testing.T) { + assert := assert.New(t) + data, err := os.ReadFile("test_responses/dms/dm_message_with_reacc.json") + if err != nil { + panic(err) + } + var api_message APIDMMessage + err = json.Unmarshal(data, &api_message) + require.NoError(t, err) + + message := ParseAPIDMMessage(api_message) + assert.Equal(message.ID, DMMessageID(1663623062195957773)) + require.Len(t, message.Reactions, 1) + + reacc := message.Reactions[0] + assert.Equal(reacc.ID, DMMessageID(1665914315742781440)) + assert.Equal(reacc.SentAt, TimestampFromUnix(1686019898732)) + assert.Equal(reacc.DMMessageID, DMMessageID(1663623062195957773)) + assert.Equal(reacc.SenderID, UserID(1458284524761075714)) + assert.Equal(reacc.Emoji, "😂") +} + +func TestParseAPIDMConversation(t *testing.T) { + assert := assert.New(t) + data, err := os.ReadFile("test_responses/dms/dm_chat_room.json") + if err != nil { + panic(err) + } + var api_room APIDMConversation + err = json.Unmarshal(data, &api_room) + require.NoError(t, err) + + // Simulate one of the participants being logged in + InitApi(API{UserID: 1458284524761075714}) + + chat_room := ParseAPIDMChatRoom(api_room) + assert.Equal(DMChatRoomID("1458284524761075714-1488963321701171204"), chat_room.ID) + assert.Equal("ONE_TO_ONE", chat_room.Type) + assert.Equal(TimestampFromUnix(1686025129086), chat_room.LastMessagedAt) + assert.False(chat_room.IsNSFW) + + assert.Len(chat_room.Participants, 2) + + p1 := chat_room.Participants[0] + assert.Equal(UserID(1458284524761075714), p1.UserID) + assert.Equal(DMMessageID(1665936253483614212), p1.LastReadEventID) + assert.True(p1.IsChatSettingsValid) + assert.False(p1.IsNotificationsDisabled) + assert.False(p1.IsReadOnly) + assert.True(p1.IsTrusted) + assert.False(p1.IsMuted) + assert.Equal("AT_END", p1.Status) + + p2 := chat_room.Participants[1] + assert.Equal(UserID(1488963321701171204), p2.UserID) + assert.Equal(DMMessageID(1663623062195957773), p2.LastReadEventID) + assert.False(p2.IsChatSettingsValid) +} diff --git a/pkg/scraper/dm_chat_room.go b/pkg/scraper/dm_chat_room.go new file mode 100644 index 0000000..f23ec81 --- /dev/null +++ b/pkg/scraper/dm_chat_room.go @@ -0,0 +1,56 @@ +package scraper + +import "fmt" + +type DMChatRoomID string + +type DMChatParticipant struct { + UserID UserID + DMChatRoomID DMChatRoomID + LastReadEventID DMMessageID + + IsChatSettingsValid bool + IsNotificationsDisabled bool + IsReadOnly bool + IsTrusted bool + IsMuted bool + Status string +} + +type DMChatRoom struct { + ID DMChatRoomID + Type string + LastMessagedAt Timestamp + IsNSFW bool + + Participants []DMChatParticipant +} + +func ParseAPIDMChatRoom(api_room APIDMConversation) DMChatRoom { + fmt.Printf("%#v\n", api_room) + ret := DMChatRoom{} + ret.ID = DMChatRoomID(api_room.ConversationID) + ret.Type = api_room.Type + ret.LastMessagedAt = TimestampFromUnix(int64(api_room.SortTimestamp)) + ret.IsNSFW = api_room.NSFW + + ret.Participants = []DMChatParticipant{} + for _, api_participant := range api_room.Participants { + participant := DMChatParticipant{} + participant.UserID = UserID(api_participant.UserID) + participant.DMChatRoomID = ret.ID + participant.LastReadEventID = DMMessageID(api_participant.LastReadEventID) + + // Process chat settings if this is the logged-in user + if participant.UserID == the_api.UserID { + participant.IsNotificationsDisabled = api_room.NotificationsDisabled + participant.IsReadOnly = api_room.ReadOnly + participant.IsTrusted = api_room.Trusted + participant.IsMuted = api_room.Muted + participant.Status = api_room.Status + participant.IsChatSettingsValid = true + } + ret.Participants = append(ret.Participants, participant) + } + return ret +} diff --git a/pkg/scraper/dm_message.go b/pkg/scraper/dm_message.go new file mode 100644 index 0000000..c55fe80 --- /dev/null +++ b/pkg/scraper/dm_message.go @@ -0,0 +1,33 @@ +package scraper + +type DMMessageID int + +type DMMessage struct { + ID DMMessageID `db:"id"` + DMChatRoomID DMChatRoomID + SenderID UserID + SentAt Timestamp + RequestID string + Text string + InReplyToID DMMessageID + Reactions []DMReaction +} + +func ParseAPIDMMessage(message APIDMMessage) DMMessage { + ret := DMMessage{} + ret.ID = DMMessageID(message.ID) + ret.SentAt = TimestampFromUnix(int64(message.Time)) + ret.DMChatRoomID = DMChatRoomID(message.ConversationID) + ret.SenderID = UserID(message.MessageData.SenderID) + ret.Text = message.MessageData.Text + + ret.InReplyToID = DMMessageID(message.MessageData.ReplyData.ID) // Will be "0" if not a reply + + ret.Reactions = []DMReaction{} + for _, api_reacc := range message.MessageReactions { + reacc := ParseAPIDMReaction(api_reacc) + reacc.DMMessageID = ret.ID + ret.Reactions = append(ret.Reactions, reacc) + } + return ret +} diff --git a/pkg/scraper/dm_reaction.go b/pkg/scraper/dm_reaction.go index 0e47b2d..74bdc4f 100644 --- a/pkg/scraper/dm_reaction.go +++ b/pkg/scraper/dm_reaction.go @@ -1,12 +1,18 @@ package scraper -type DMReactionID int - type DMReaction struct { - ID DMReactionID `db:"id"` - Time int - ConversationID ConversationID - MessageID DMID - ReactionKey string - SenderID UserID + ID DMMessageID `db:"id"` + DMMessageID DMMessageID + SenderID UserID + SentAt Timestamp + Emoji string +} + +func ParseAPIDMReaction(reacc APIDMReaction) DMReaction { + ret := DMReaction{} + ret.ID = DMMessageID(reacc.ID) + ret.SenderID = UserID(reacc.SenderID) + ret.SentAt = TimestampFromUnix(int64(reacc.Time)) + ret.Emoji = reacc.Emoji + return ret }