diff --git a/scraper/api_graphql_utils.go b/scraper/api_graphql_utils.go new file mode 100644 index 0000000..39bead8 --- /dev/null +++ b/scraper/api_graphql_utils.go @@ -0,0 +1,85 @@ +package scraper + +import ( + "encoding/json" + "net/url" +) + +type GraphqlVariables struct { + UserID UserID `json:"userId,string,omitempty"` + FocalTweetID TweetID `json:"focalTweetId,string,omitempty"` + Cursor string `json:"cursor,omitempty"` + WithRuxInjections bool `json:"with_rux_injections"` + IncludePromotedContent bool `json:"includePromotedContent"` + Count int `json:"count,omitempty"` + WithCommunity bool `json:"withCommunity"` + WithQuickPromoteEligibilityTweetFields bool `json:"withQuickPromoteEligibilityTweetFields"` + WithSuperFollowsUserFields bool `json:"withSuperFollowsUserFields,omitempty"` + WithBirdwatchPivots bool `json:"withBirdwatchPivots"` + WithBirdwatchNotes bool `json:"withBirdwatchNotes,omitempty"` + WithDownvotePerspective bool `json:"withDownvotePerspective"` + WithReactionsMetadata bool `json:"withReactionsMetadata"` + WithReactionsPerspective bool `json:"withReactionsPerspective"` + WithSuperFollowsTweetFields bool `json:"withSuperFollowsTweetFields,omitempty"` + WithVoice bool `json:"withVoice"` + WithV2Timeline bool `json:"withV2Timeline"` + FSInteractiveText bool `json:"__fs_interactive_text,omitempty"` + FSResponsiveWebUCGqlEnabled bool `json:"__fs_responsive_web_uc_gql_enabled,omitempty"` + FSDontMentionMeViewApiEnabled bool `json:"__fs_dont_mention_me_view_api_enabled,omitempty"` +} + +type GraphqlFeatures struct { + ResponsiveWebTwitterBlueVerifiedBadgeIsEnabled bool `json:"responsive_web_twitter_blue_verified_badge_is_enabled,omitempty"` //nolint:lll // I didn't choose this field name + RWebListsTimelineRedesignEnabled bool `json:"rweb_lists_timeline_redesign_enabled"` + ResponsiveWebGraphqlExcludeDirectiveEnabled bool `json:"responsive_web_graphql_exclude_directive_enabled"` + VerifiedPhoneLabelEnabled bool `json:"verified_phone_label_enabled"` + CreatorSubscriptionsTweetPreviewApiEnabled bool `json:"creator_subscriptions_tweet_preview_api_enabled"` + ResponsiveWebGraphqlTimelineNavigationEnabled bool `json:"responsive_web_graphql_timeline_navigation_enabled"` + ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled bool `json:"responsive_web_graphql_skip_user_profile_image_extensions_enabled"` //nolint:lll // I didn't choose this field name + TweetypieUnmentionOptimizationEnabled bool `json:"tweetypie_unmention_optimization_enabled"` + ResponsiveWebEditTweetApiEnabled bool `json:"responsive_web_edit_tweet_api_enabled"` + GraphqlIsTranslatableRWebTweetIsTranslatableEnabled bool `json:"graphql_is_translatable_rweb_tweet_is_translatable_enabled"` + ViewCountsEverywhereApiEnabled bool `json:"view_counts_everywhere_api_enabled"` + LongformNotetweetsConsumptionEnabled bool `json:"longform_notetweets_consumption_enabled"` + TweetAwardsWebTippingEnabled bool `json:"tweet_awards_web_tipping_enabled"` + FreedomOfSpeechNotReachFetchEnabled bool `json:"freedom_of_speech_not_reach_fetch_enabled"` + StandardizedNudgesMisinfo bool `json:"standardized_nudges_misinfo"` + TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled bool `json:"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled"` //nolint:lll // I didn't choose this field name + LongformNotetweetsRichTextReadEnabled bool `json:"longform_notetweets_rich_text_read_enabled"` + LongformNotetweetsInlineMediaEnabled bool `json:"longform_notetweets_inline_media_enabled"` + ResponsiveWebEnhanceCardsEnabled bool `json:"responsive_web_enhance_cards_enabled"` + UnifiedCardsAdMetadataContainerDynamicCardContentQueryEnabled bool `json:"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled,omitempty"` //nolint:lll // I didn't choose this field name + ResponsiveWebUcGqlEnabled bool `json:"responsive_web_uc_gql_enabled,omitempty"` + VibeApiEnabled bool `json:"vibe_api_enabled,omitempty"` + InteractiveTextEnabled bool `json:"interactive_text_enabled,omitempty"` + ResponsiveWebTextConversationsEnabled bool `json:"responsive_web_text_conversations_enabled,omitempty"` +} + +type GraphqlURL struct { + BaseUrl string + Variables GraphqlVariables + Features GraphqlFeatures +} + +func (u GraphqlURL) String() string { + features_bytes, err := json.Marshal(u.Features) + if err != nil { + panic(err) + } + vars_bytes, err := json.Marshal(u.Variables) + if err != nil { + panic(err) + } + + ret, err := url.Parse(u.BaseUrl) + if err != nil { + panic(err) + } + q := ret.Query() + if u.Features != (GraphqlFeatures{}) { + q.Add("features", string(features_bytes)) + } + q.Add("variables", string(vars_bytes)) + ret.RawQuery = q.Encode() + return ret.String() +} diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index ce68d1b..9827c57 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -678,18 +678,62 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { } func get_graphql_user_timeline_url(user_id UserID, cursor string) string { - if cursor != "" { - return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" //nolint:lll // It's a URL, come on - } - return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Afalse%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" //nolint:lll // It's a URL, come on + return GraphqlURL{ + BaseUrl: "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies", + Variables: GraphqlVariables{ + UserID: user_id, + Count: 40, + Cursor: cursor, + IncludePromotedContent: false, + WithCommunity: true, + WithSuperFollowsUserFields: true, + WithBirdwatchPivots: false, + WithDownvotePerspective: false, + WithReactionsMetadata: false, + WithReactionsPerspective: false, + WithSuperFollowsTweetFields: true, + WithBirdwatchNotes: false, + WithVoice: true, + WithV2Timeline: false, + }, + }.String() } func get_tweet_detail_url(tweet_id TweetID, cursor string) string { - maybe_cursor := "" - if cursor != "" { - maybe_cursor = "%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C" - } - return "https://twitter.com/i/api/graphql/tPRAv4UnqM9dOgDWggph7Q/TweetDetail?variables=%7B%22focalTweetId%22%3A%22" + fmt.Sprint(tweet_id) + "%22%2C" + maybe_cursor + "%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_lists_timeline_redesign_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D" //nolint:lll // It's a URL, come on + return GraphqlURL{ + BaseUrl: "https://twitter.com/i/api/graphql/tPRAv4UnqM9dOgDWggph7Q/TweetDetail", + Variables: GraphqlVariables{ + FocalTweetID: tweet_id, + Cursor: cursor, + WithRuxInjections: false, + IncludePromotedContent: true, // TODO: false + WithCommunity: true, + WithQuickPromoteEligibilityTweetFields: true, + WithBirdwatchNotes: true, + WithVoice: true, + WithV2Timeline: true, + }, + Features: GraphqlFeatures{ + RWebListsTimelineRedesignEnabled: true, + ResponsiveWebGraphqlExcludeDirectiveEnabled: true, + VerifiedPhoneLabelEnabled: false, + CreatorSubscriptionsTweetPreviewApiEnabled: true, + ResponsiveWebGraphqlTimelineNavigationEnabled: true, + ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false, + TweetypieUnmentionOptimizationEnabled: true, + ResponsiveWebEditTweetApiEnabled: true, + GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true, + ViewCountsEverywhereApiEnabled: true, + LongformNotetweetsConsumptionEnabled: true, + TweetAwardsWebTippingEnabled: false, + FreedomOfSpeechNotReachFetchEnabled: true, + StandardizedNudgesMisinfo: true, + TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: false, + LongformNotetweetsRichTextReadEnabled: true, + LongformNotetweetsInlineMediaEnabled: false, + ResponsiveWebEnhanceCardsEnabled: false, + }, + }.String() } /**