Add command to fetch a user's DM inbox and hook it up

cmd/twitter/main.go

@@ -75,8 +75,8 @@ func main() {
 	log.SetLevel(logging_level)
 
 	if len(args) < 2 {
-		if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" ||
-			args[0] == "fetch_timeline" || args[0] == "fetch_timeline_for_you") {
+		if len(args) == 1 && (args[0] == "list_followed" || args[0] == "webserver" || args[0] == "fetch_timeline" ||
+		                      args[0] == "fetch_timeline_for_you" || args[0] == "fetch_inbox") {
 			// Doesn't need a target, so create a fake second arg
 			args = append(args, "")
 		} else {
@@ -159,6 +159,8 @@ func main() {
 		unlike_tweet(target)
 	case "webserver":
 		start_webserver(*addr)
+	case "fetch_inbox":
+		fetch_inbox()
 	default:
 		die(fmt.Sprintf("Invalid operation: %s", operation), true, 3)
 	}
@@ -251,7 +253,6 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
 		die(err.Error(), false, -1)
 	}
 
-	//trove, err := scraper.GetTweetFull(tweet_id, how_many)
 	trove, err := scraper.GetTweetFullAPIV2(tweet_id, how_many)
 	if err != nil {
 		die(err.Error(), false, -1)
@@ -396,3 +397,9 @@ func start_webserver(addr string) {
 	app := webserver.NewApp(profile)
 	app.Run(addr)
 }
+
+func fetch_inbox() {
+	trove := scraper.GetInbox()
+	profile.SaveDMTrove(trove)
+	happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
+}

pkg/persistence/dm_trove_queries.go

@@ -0,0 +1,26 @@
+package persistence
+
+import (
+	"fmt"
+
+	. "offline_twitter/scraper"
+)
+
+// Convenience function that saves all the objects in a TweetTrove.
+// Panics if anything goes wrong.
+func (p Profile) SaveDMTrove(trove DMTrove) {
+	p.SaveTweetTrove(trove.TweetTrove)
+
+	for _, r := range trove.Rooms {
+		err := p.SaveChatRoom(r)
+		if err != nil {
+			panic(fmt.Errorf("Error saving chat room: %#v\n  %w", r, err))
+		}
+	}
+	for _, m := range trove.Messages {
+		err := p.SaveChatMessage(m)
+		if err != nil {
+			panic(fmt.Errorf("Error saving chat message: %#v\n  %w", m, err))
+		}
+	}
+}

pkg/scraper/api_types_dms.go

@@ -1,5 +1,9 @@
 package scraper
 
+import (
+	"net/url"
+)
+
 type APIDMReaction struct {
 	ID       int    `json:"id,string"`
 	Time     int    `json:"time,string"`
@@ -76,3 +80,49 @@ func (r APIDMResponse) ToDMTrove() DMTrove {
 	}
 	return ret
 }
+
+func (api *API) GetDMInbox() (APIDMResponse, error) {
+	url, err := url.Parse("https://twitter.com/i/api/1.1/dm/inbox_initial_state.json")
+	if err != nil {
+		panic(err)
+	}
+	query := url.Query()
+	query.Add("nsfw_filtering_enabled", "false")
+	query.Add("filter_low_quality", "true")
+	query.Add("include_quality", "all")
+	query.Add("include_profile_interstitial_type", "1")
+	query.Add("include_blocking", "1")
+	query.Add("include_blocked_by", "1")
+	query.Add("include_followed_by", "1")
+	query.Add("include_want_retweets", "1")
+	query.Add("include_mute_edge", "1")
+	query.Add("include_can_dm", "1")
+	query.Add("include_can_media_tag", "1")
+	query.Add("include_ext_has_nft_avatar", "1")
+	query.Add("include_ext_is_blue_verified", "1")
+	query.Add("include_ext_verified_type", "1")
+	query.Add("include_ext_profile_image_shape", "1")
+	query.Add("skip_status", "1")
+	query.Add("dm_secret_conversations_enabled", "false")
+	query.Add("krs_registration_enabled", "true")
+	query.Add("cards_platform", "Web-12")
+	query.Add("include_cards", "1")
+	query.Add("include_ext_alt_text", "true")
+	query.Add("include_ext_limited_action_results", "false")
+	query.Add("include_quote_count", "true")
+	query.Add("include_reply_count", "1")
+	query.Add("tweet_mode", "extended")
+	query.Add("include_ext_views", "true")
+	query.Add("dm_users", "true")
+	query.Add("include_groups", "true")
+	query.Add("include_inbox_timelines", "true")
+	query.Add("include_ext_media_color", "true")
+	query.Add("supports_reactions", "true")
+	query.Add("include_ext_edit_control", "true")
+	query.Add("ext", "mediaColor,altText,mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,vibe")
+	url.RawQuery = query.Encode()
+
+	var result APIDMResponse
+	err = api.do_http(url.String(), "", &result)
+	return result, err
+}

scraper/dm_trove.go

@@ -1,5 +1,9 @@
 package scraper
 
+import (
+	log "github.com/sirupsen/logrus"
+)
+
 type DMTrove struct {
 	Rooms      map[DMChatRoomID]DMChatRoom
 	Messages   map[DMMessageID]DMMessage
@@ -23,3 +27,14 @@ func (t1 *DMTrove) MergeWith(t2 DMTrove) {
 	}
 	t1.TweetTrove.MergeWith(t2.TweetTrove)
 }
+
+func GetInbox() DMTrove {
+	if !the_api.IsAuthenticated {
+		log.Fatalf("Fetching DMs can only be done when authenticated.  Please provide `--session [user]`")
+	}
+	dm_response, err := the_api.GetDMInbox()
+	if err != nil {
+		panic(err)
+	}
+	return dm_response.ToDMTrove()
+}