Create structures for GraphQL request variables and features

scraper/api_graphql_utils.go

@@ -0,0 +1,85 @@
+package scraper
+
+import (
+	"encoding/json"
+	"net/url"
+)
+
+type GraphqlVariables struct {
+	UserID                                 UserID  `json:"userId,string,omitempty"`
+	FocalTweetID                           TweetID `json:"focalTweetId,string,omitempty"`
+	Cursor                                 string  `json:"cursor,omitempty"`
+	WithRuxInjections                      bool    `json:"with_rux_injections"`
+	IncludePromotedContent                 bool    `json:"includePromotedContent"`
+	Count                                  int     `json:"count,omitempty"`
+	WithCommunity                          bool    `json:"withCommunity"`
+	WithQuickPromoteEligibilityTweetFields bool    `json:"withQuickPromoteEligibilityTweetFields"`
+	WithSuperFollowsUserFields             bool    `json:"withSuperFollowsUserFields,omitempty"`
+	WithBirdwatchPivots                    bool    `json:"withBirdwatchPivots"`
+	WithBirdwatchNotes                     bool    `json:"withBirdwatchNotes,omitempty"`
+	WithDownvotePerspective                bool    `json:"withDownvotePerspective"`
+	WithReactionsMetadata                  bool    `json:"withReactionsMetadata"`
+	WithReactionsPerspective               bool    `json:"withReactionsPerspective"`
+	WithSuperFollowsTweetFields            bool    `json:"withSuperFollowsTweetFields,omitempty"`
+	WithVoice                              bool    `json:"withVoice"`
+	WithV2Timeline                         bool    `json:"withV2Timeline"`
+	FSInteractiveText                      bool    `json:"__fs_interactive_text,omitempty"`
+	FSResponsiveWebUCGqlEnabled            bool    `json:"__fs_responsive_web_uc_gql_enabled,omitempty"`
+	FSDontMentionMeViewApiEnabled          bool    `json:"__fs_dont_mention_me_view_api_enabled,omitempty"`
+}
+
+type GraphqlFeatures struct {
+	ResponsiveWebTwitterBlueVerifiedBadgeIsEnabled                 bool `json:"responsive_web_twitter_blue_verified_badge_is_enabled,omitempty"` //nolint:lll // I didn't choose this field name
+	RWebListsTimelineRedesignEnabled                               bool `json:"rweb_lists_timeline_redesign_enabled"`
+	ResponsiveWebGraphqlExcludeDirectiveEnabled                    bool `json:"responsive_web_graphql_exclude_directive_enabled"`
+	VerifiedPhoneLabelEnabled                                      bool `json:"verified_phone_label_enabled"`
+	CreatorSubscriptionsTweetPreviewApiEnabled                     bool `json:"creator_subscriptions_tweet_preview_api_enabled"`
+	ResponsiveWebGraphqlTimelineNavigationEnabled                  bool `json:"responsive_web_graphql_timeline_navigation_enabled"`
+	ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled      bool `json:"responsive_web_graphql_skip_user_profile_image_extensions_enabled"` //nolint:lll // I didn't choose this field name
+	TweetypieUnmentionOptimizationEnabled                          bool `json:"tweetypie_unmention_optimization_enabled"`
+	ResponsiveWebEditTweetApiEnabled                               bool `json:"responsive_web_edit_tweet_api_enabled"`
+	GraphqlIsTranslatableRWebTweetIsTranslatableEnabled            bool `json:"graphql_is_translatable_rweb_tweet_is_translatable_enabled"`
+	ViewCountsEverywhereApiEnabled                                 bool `json:"view_counts_everywhere_api_enabled"`
+	LongformNotetweetsConsumptionEnabled                           bool `json:"longform_notetweets_consumption_enabled"`
+	TweetAwardsWebTippingEnabled                                   bool `json:"tweet_awards_web_tipping_enabled"`
+	FreedomOfSpeechNotReachFetchEnabled                            bool `json:"freedom_of_speech_not_reach_fetch_enabled"`
+	StandardizedNudgesMisinfo                                      bool `json:"standardized_nudges_misinfo"`
+	TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled bool `json:"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled"` //nolint:lll // I didn't choose this field name
+	LongformNotetweetsRichTextReadEnabled                          bool `json:"longform_notetweets_rich_text_read_enabled"`
+	LongformNotetweetsInlineMediaEnabled                           bool `json:"longform_notetweets_inline_media_enabled"`
+	ResponsiveWebEnhanceCardsEnabled                               bool `json:"responsive_web_enhance_cards_enabled"`
+	UnifiedCardsAdMetadataContainerDynamicCardContentQueryEnabled  bool `json:"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled,omitempty"` //nolint:lll // I didn't choose this field name
+	ResponsiveWebUcGqlEnabled                                      bool `json:"responsive_web_uc_gql_enabled,omitempty"`
+	VibeApiEnabled                                                 bool `json:"vibe_api_enabled,omitempty"`
+	InteractiveTextEnabled                                         bool `json:"interactive_text_enabled,omitempty"`
+	ResponsiveWebTextConversationsEnabled                          bool `json:"responsive_web_text_conversations_enabled,omitempty"`
+}
+
+type GraphqlURL struct {
+	BaseUrl   string
+	Variables GraphqlVariables
+	Features  GraphqlFeatures
+}
+
+func (u GraphqlURL) String() string {
+	features_bytes, err := json.Marshal(u.Features)
+	if err != nil {
+		panic(err)
+	}
+	vars_bytes, err := json.Marshal(u.Variables)
+	if err != nil {
+		panic(err)
+	}
+
+	ret, err := url.Parse(u.BaseUrl)
+	if err != nil {
+		panic(err)
+	}
+	q := ret.Query()
+	if u.Features != (GraphqlFeatures{}) {
+		q.Add("features", string(features_bytes))
+	}
+	q.Add("variables", string(vars_bytes))
+	ret.RawQuery = q.Encode()
+	return ret.String()
+}

scraper/api_types_v2.go

@@ -678,18 +678,62 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
 }
 
 func get_graphql_user_timeline_url(user_id UserID, cursor string) string {
-	if cursor != "" {
+	return GraphqlURL{
-		return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" //nolint:lll  // It's a URL, come on
+		BaseUrl: "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies",
-	}
+		Variables: GraphqlVariables{
-	return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Afalse%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" //nolint:lll  // It's a URL, come on
+			UserID:                      user_id,
+			Count:                       40,
+			Cursor:                      cursor,
+			IncludePromotedContent:      false,
+			WithCommunity:               true,
+			WithSuperFollowsUserFields:  true,
+			WithBirdwatchPivots:         false,
+			WithDownvotePerspective:     false,
+			WithReactionsMetadata:       false,
+			WithReactionsPerspective:    false,
+			WithSuperFollowsTweetFields: true,
+			WithBirdwatchNotes:          false,
+			WithVoice:                   true,
+			WithV2Timeline:              false,
+		},
+	}.String()
 }
 
 func get_tweet_detail_url(tweet_id TweetID, cursor string) string {
-	maybe_cursor := ""
+	return GraphqlURL{
-	if cursor != "" {
+		BaseUrl: "https://twitter.com/i/api/graphql/tPRAv4UnqM9dOgDWggph7Q/TweetDetail",
-		maybe_cursor = "%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C"
+		Variables: GraphqlVariables{
-	}
+			FocalTweetID:                           tweet_id,
-	return "https://twitter.com/i/api/graphql/tPRAv4UnqM9dOgDWggph7Q/TweetDetail?variables=%7B%22focalTweetId%22%3A%22" + fmt.Sprint(tweet_id) + "%22%2C" + maybe_cursor + "%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D" //nolint:lll  // It's a URL, come on
+			Cursor:                                 cursor,
+			WithRuxInjections:                      false,
+			IncludePromotedContent:                 true, // TODO: false
+			WithCommunity:                          true,
+			WithQuickPromoteEligibilityTweetFields: true,
+			WithBirdwatchNotes:                     true,
+			WithVoice:                              true,
+			WithV2Timeline:                         true,
+		},
+		Features: GraphqlFeatures{
+			RWebListsTimelineRedesignEnabled:                               true,
+			ResponsiveWebGraphqlExcludeDirectiveEnabled:                    true,
+			VerifiedPhoneLabelEnabled:                                      false,
+			CreatorSubscriptionsTweetPreviewApiEnabled:                     true,
+			ResponsiveWebGraphqlTimelineNavigationEnabled:                  true,
+			ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled:      false,
+			TweetypieUnmentionOptimizationEnabled:                          true,
+			ResponsiveWebEditTweetApiEnabled:                               true,
+			GraphqlIsTranslatableRWebTweetIsTranslatableEnabled:            true,
+			ViewCountsEverywhereApiEnabled:                                 true,
+			LongformNotetweetsConsumptionEnabled:                           true,
+			TweetAwardsWebTippingEnabled:                                   false,
+			FreedomOfSpeechNotReachFetchEnabled:                            true,
+			StandardizedNudgesMisinfo:                                      true,
+			TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: false,
+			LongformNotetweetsRichTextReadEnabled:                          true,
+			LongformNotetweetsInlineMediaEnabled:                           false,
+			ResponsiveWebEnhanceCardsEnabled:                               false,
+		},
+	}.String()
 }
 
 /**