From 57500b0d30349a027e6a908bd2bda63708949689 Mon Sep 17 00:00:00 2001 From: Alessio Date: Fri, 23 Jun 2023 15:54:29 -0300 Subject: [PATCH] Add command to fetch a user's DM inbox and hook it up --- cmd/twitter/main.go | 13 ++++++-- pkg/persistence/dm_trove_queries.go | 26 +++++++++++++++ pkg/scraper/api_types_dms.go | 50 +++++++++++++++++++++++++++++ scraper/dm_trove.go | 15 +++++++++ 4 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 pkg/persistence/dm_trove_queries.go diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 5cc905a..f5c64d9 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -75,8 +75,8 @@ func main() { log.SetLevel(logging_level) if len(args) < 2 { - if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" || - args[0] == "fetch_timeline" || args[0] == "fetch_timeline_for_you") { + if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" || args[0] == "fetch_timeline" || + args[0] == "fetch_timeline_for_you" || args[0] == "fetch_inbox") { // Doesn't need a target, so create a fake second arg args = append(args, "") } else { @@ -159,6 +159,8 @@ func main() { unlike_tweet(target) case "webserver": start_webserver(*addr) + case "fetch_inbox": + fetch_inbox() default: die(fmt.Sprintf("Invalid operation: %s", operation), true, 3) } @@ -251,7 +253,6 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) { die(err.Error(), false, -1) } - //trove, err := scraper.GetTweetFull(tweet_id, how_many) trove, err := scraper.GetTweetFullAPIV2(tweet_id, how_many) if err != nil { die(err.Error(), false, -1) @@ -396,3 +397,9 @@ func start_webserver(addr string) { app := webserver.NewApp(profile) app.Run(addr) } + +func fetch_inbox() { + trove := scraper.GetInbox() + profile.SaveDMTrove(trove) + happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms))) +} diff --git a/pkg/persistence/dm_trove_queries.go b/pkg/persistence/dm_trove_queries.go new file mode 100644 index 0000000..d638e0c --- /dev/null +++ b/pkg/persistence/dm_trove_queries.go @@ -0,0 +1,26 @@ +package persistence + +import ( + "fmt" + + . "offline_twitter/scraper" +) + +// Convenience function that saves all the objects in a TweetTrove. +// Panics if anything goes wrong. +func (p Profile) SaveDMTrove(trove DMTrove) { + p.SaveTweetTrove(trove.TweetTrove) + + for _, r := range trove.Rooms { + err := p.SaveChatRoom(r) + if err != nil { + panic(fmt.Errorf("Error saving chat room: %#v\n %w", r, err)) + } + } + for _, m := range trove.Messages { + err := p.SaveChatMessage(m) + if err != nil { + panic(fmt.Errorf("Error saving chat message: %#v\n %w", m, err)) + } + } +} diff --git a/pkg/scraper/api_types_dms.go b/pkg/scraper/api_types_dms.go index cc9c048..c962da0 100644 --- a/pkg/scraper/api_types_dms.go +++ b/pkg/scraper/api_types_dms.go @@ -1,5 +1,9 @@ package scraper +import ( + "net/url" +) + type APIDMReaction struct { ID int `json:"id,string"` Time int `json:"time,string"` @@ -76,3 +80,49 @@ func (r APIDMResponse) ToDMTrove() DMTrove { } return ret } + +func (api *API) GetDMInbox() (APIDMResponse, error) { + url, err := url.Parse("https://twitter.com/i/api/1.1/dm/inbox_initial_state.json") + if err != nil { + panic(err) + } + query := url.Query() + query.Add("nsfw_filtering_enabled", "false") + query.Add("filter_low_quality", "true") + query.Add("include_quality", "all") + query.Add("include_profile_interstitial_type", "1") + query.Add("include_blocking", "1") + query.Add("include_blocked_by", "1") + query.Add("include_followed_by", "1") + query.Add("include_want_retweets", "1") + query.Add("include_mute_edge", "1") + query.Add("include_can_dm", "1") + query.Add("include_can_media_tag", "1") + query.Add("include_ext_has_nft_avatar", "1") + query.Add("include_ext_is_blue_verified", "1") + query.Add("include_ext_verified_type", "1") + query.Add("include_ext_profile_image_shape", "1") + query.Add("skip_status", "1") + query.Add("dm_secret_conversations_enabled", "false") + query.Add("krs_registration_enabled", "true") + query.Add("cards_platform", "Web-12") + query.Add("include_cards", "1") + query.Add("include_ext_alt_text", "true") + query.Add("include_ext_limited_action_results", "false") + query.Add("include_quote_count", "true") + query.Add("include_reply_count", "1") + query.Add("tweet_mode", "extended") + query.Add("include_ext_views", "true") + query.Add("dm_users", "true") + query.Add("include_groups", "true") + query.Add("include_inbox_timelines", "true") + query.Add("include_ext_media_color", "true") + query.Add("supports_reactions", "true") + query.Add("include_ext_edit_control", "true") + query.Add("ext", "mediaColor,altText,mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,vibe") + url.RawQuery = query.Encode() + + var result APIDMResponse + err = api.do_http(url.String(), "", &result) + return result, err +} diff --git a/scraper/dm_trove.go b/scraper/dm_trove.go index 7bbe866..c8f4444 100644 --- a/scraper/dm_trove.go +++ b/scraper/dm_trove.go @@ -1,5 +1,9 @@ package scraper +import ( + log "github.com/sirupsen/logrus" +) + type DMTrove struct { Rooms map[DMChatRoomID]DMChatRoom Messages map[DMMessageID]DMMessage @@ -23,3 +27,14 @@ func (t1 *DMTrove) MergeWith(t2 DMTrove) { } t1.TweetTrove.MergeWith(t2.TweetTrove) } + +func GetInbox() DMTrove { + if !the_api.IsAuthenticated { + log.Fatalf("Fetching DMs can only be done when authenticated. Please provide `--session [user]`") + } + dm_response, err := the_api.GetDMInbox() + if err != nil { + panic(err) + } + return dm_response.ToDMTrove() +}