diff --git a/doc/curl requests b/doc/curl requests index 12cc29e..5a4210d 100644 --- a/doc/curl requests +++ b/doc/curl requests @@ -53,3 +53,22 @@ curl \ -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ -H "X-Guest-Token: 1591520847784706050" \ "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%221BdxYypQzBgxX%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D" + + +# +# send a message +curl 'https://twitter.com/i/api/1.1/dm/new2.json?ext=mediaColor%2CaltText%2CmediaStats%2ChighlightedLabel%2ChasNftAvatar%2CvoiceInfo%2CbirdwatchPivot%2Cenrichments%2CsuperFollowMetadata%2CunmentionInfo%2CeditControl%2Cvibe&include_ext_alt_text=true&include_ext_limited_action_results=false&include_reply_count=1&tweet_mode=extended&include_ext_views=true&include_groups=true&include_inbox_timelines=true&include_ext_media_color=true&supports_reactions=true' \ + -H 'authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' \ + --data-raw '{"conversation_id":"1458284524761075714-1648469647187124225","recipient_ids":false,"request_id":"e9795640-fe77-11ed-8430-3b5d1c154555","text":"ignore this","cards_platform":"Web-12","include_cards":1,"include_quote_count":true,"dm_users":false}' \ + --compressed ; + +# +# Fetch snapshot of messages +curl 'https://twitter.com/i/api/1.1/dm/conversation/1458284524761075714-1648469647187124225.json?max_id=1663307019661574149&context=FETCH_DM_CONVERSATION_HISTORY&include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&include_ext_has_nft_avatar=1&include_ext_is_blue_verified=1&include_ext_verified_type=1&include_ext_profile_image_shape=1&skip_status=1&dm_secret_conversations_enabled=false&krs_registration_enabled=true&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_ext_limited_action_results=false&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_ext_views=true&dm_users=false&include_groups=true&include_inbox_timelines=true&include_ext_media_color=true&supports_reactions=true&include_conversation_info=true&ext=mediaColor%2CaltText%2CmediaStats%2ChighlightedLabel%2ChasNftAvatar%2CvoiceInfo%2CbirdwatchPivot%2Cenrichments%2CsuperFollowMetadata%2CunmentionInfo%2CeditControl%2Cvibe' \ +-H 'authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' \ +-H 'cookie: kdt=uiBKvHgG2m5hFLQc8ZU8XsWOWM25MqaMKJJEb3oE; lang=en; lv-uid=AAAAEICX4yaMhNgtfux4BknjQW7rGOkl2l0_MGnFIKDnNIRlbQn9Ji-kab-WgufG; dnt=1; auth_multi="1590450335315021824:c018f82b02685aaefee2be4ce1b18c35eaee483b"; auth_token=3dd34d00d684eea1645df3dafd0c728464bc284c; guest_id=v1%3A168539432661461192; ct0=ba56cf7397b4a071f491b73e60c643829e55f37a26537ba34e96dba5419ad9a9b9f4dba81a19a4a4a0468c6bebc004e0d08a41fc90b06568c6c1712ede38ffb0d919c0c39bc695ceb4a2dc8190cf3936; twid=u%3D1648469647187124225;"' \ +-H 'x-client-uuid: d195047c-6532-40a3-b494-957f4fd2ab3d' \ +-H 'x-twitter-active-user: yes' \ +-H 'x-twitter-auth-type: OAuth2Session' \ +-H 'x-twitter-client-language: en' \ +--compressed \ No newline at end of file diff --git a/pkg/persistence/schema.sql b/pkg/persistence/schema.sql index 3c3dd8f..4edcbc6 100644 --- a/pkg/persistence/schema.sql +++ b/pkg/persistence/schema.sql @@ -206,6 +206,42 @@ create index if not exists index_likes_tweet_id on likes (tweet_id); create table fake_user_sequence(latest_fake_id integer not null); insert into fake_user_sequence values(0x4000000000000000); +create table conversations (rowid integer primary key, + conversation_id text unique not null check(typeof(id) = 'text'), + type text not null, + sort_event_id int, + sort_timestamp int, + foreign key(participants) references users(id) + nsfw boolean not null, + notifications_disabled boolean not null, + mention_notifications_disabled boolean not null, + last_read_event_id int not null unique, + read_only boolean not null, + trusted boolean not null, + low_quality boolean not null, + muted boolean not null, +) + +create table messages (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + time integer not null, + request_id integer not null, + foreign key(conversation_id) references conversations(id) + foreign key(recipient_id) references users(id) + foreign key(sender_id) references users(id) + text text not null + foreign key(message_reactions) references message_reactions(id) +); + +create table message_reactions (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + time integer not null, + foreign key(conversation_id) references conversations(id) + foreign key(message_id) references messages(id) + reaction_key text not null, + foreign key(sender_id) references users(id) +) + create table database_version(rowid integer primary key, version_number integer not null unique ); diff --git a/pkg/scraper/api_types_notifications.go b/pkg/scraper/api_types_notifications.go new file mode 100644 index 0000000..a649ca8 --- /dev/null +++ b/pkg/scraper/api_types_notifications.go @@ -0,0 +1,21 @@ +package scraper + +import ( + "net/url" +) + +func (api API) GetNotifications(cursor string) (TweetResponse, error) { + url, err := url.Parse("https://api.twitter.com/2/notifications/all.json") + if err != nil { + panic(err) + } + + query := url.Query() + add_tweet_query_params(&query) + url.RawQuery = query.Encode() + + var result TweetResponse + err = api.do_http(url.String(), cursor, &result) + + return result, err +} diff --git a/pkg/scraper/conversation.go b/pkg/scraper/conversation.go new file mode 100644 index 0000000..4d705d1 --- /dev/null +++ b/pkg/scraper/conversation.go @@ -0,0 +1,18 @@ +package scraper + +type ConversationID string + +type Conversation struct { + ID ConversationID + Type string + SortEventID int + SortTimestamp int + Participants []User + Nsfw bool + NotificationsDisabled bool + LastReadEventId int + ReadOnly bool + Trusted bool + LowQuality bool + Muted bool +} diff --git a/pkg/scraper/dm.go b/pkg/scraper/dm.go new file mode 100644 index 0000000..4771cd2 --- /dev/null +++ b/pkg/scraper/dm.go @@ -0,0 +1,14 @@ +package scraper + +type DMID string + +type DM struct { + ID DMID `db:"id"` + Time int + Request int + ConversationID ConversationID + RecipientID UserID + SenderID UserID + Text string + MessageReactions []DMReaction +} diff --git a/pkg/scraper/dm_reaction.go b/pkg/scraper/dm_reaction.go new file mode 100644 index 0000000..0e47b2d --- /dev/null +++ b/pkg/scraper/dm_reaction.go @@ -0,0 +1,12 @@ +package scraper + +type DMReactionID int + +type DMReaction struct { + ID DMReactionID `db:"id"` + Time int + ConversationID ConversationID + MessageID DMID + ReactionKey string + SenderID UserID +} diff --git a/pkg/scraper/list.go b/pkg/scraper/list.go new file mode 100644 index 0000000..64077b2 --- /dev/null +++ b/pkg/scraper/list.go @@ -0,0 +1,12 @@ +package scraper + +type ListID int + +type List struct { + ID ListID `db:"rowid"` + Type string `db:"type"` + Name string `db:"name"` + + UserIDs []UserID + Users []*User +}