From 8a70f55e1b273b76e9722863465fddcc998d553f Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 14 Feb 2022 17:15:01 -0800 Subject: [PATCH] Update some more stuff to match API changes - set X-Twitter-Client-Language header to "en" - Add a new tombstone type (age-restricted) --- doc/curl requests | 14 +++++++++++++- persistence/schema.sql | 3 ++- persistence/versions.go | 3 ++- scraper/api_request_utils.go | 1 + scraper/api_types.go | 1 + 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/curl requests b/doc/curl requests index 83424a0..b431a2c 100644 --- a/doc/curl requests +++ b/doc/curl requests @@ -13,11 +13,23 @@ curl \ # # A user's feed: -curl \ +curl -i \ -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ -H "X-Guest-Token: 1396177150890348547" \ https://api.twitter.com/2/timeline/profile/44067298.json +# +# User feed APIv2: + +curl -i \ + -H 'authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' \ + -H 'x-csrf-token: 8e73dca13c4f04c3a9136b6b6204d9e9809e8f544bec6deb4cf1f5054f0c5d54b332433b5dcd2b3f7a5c6fa241e567beeebeff22a3f528fa44b0e6fc1ad75bf0a7cb7bc5b0be1a01a9d5e3e40cd19b69' \ + -H 'cookie: ct0=8e73dca13c4f04c3a9136b6b6204d9e9809e8f544bec6deb4cf1f5054f0c5d54b332433b5dcd2b3f7a5c6fa241e567beeebeff22a3f528fa44b0e6fc1ad75bf0a7cb7bc5b0be1a01a9d5e3e40cd19b69' \ + -H 'x-guest-token: 1493389889173291009' \ + -H 'x-twitter-client-language: en' \ + 'https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22599817378%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Afalse%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D' + + # # A tweet and replies (conversation): diff --git a/persistence/schema.sql b/persistence/schema.sql index 7045ebb..12bde6d 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -32,7 +32,8 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values (3, 'hidden', 'You’re unable to view this Tweet because this account owner limits who can view their Tweets'), (4, 'unavailable', 'This Tweet is unavailable'), (5, 'violated', 'This Tweet violated the Twitter Rules'), - (6, 'no longer exists', 'This Tweet is from an account that no longer exists'); + (6, 'no longer exists', 'This Tweet is from an account that no longer exists'), + (7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter'); create table tweets (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), diff --git a/persistence/versions.go b/persistence/versions.go index f105f30..daa9d71 100644 --- a/persistence/versions.go +++ b/persistence/versions.go @@ -8,7 +8,7 @@ import ( ) -const ENGINE_DATABASE_VERSION = 7 +const ENGINE_DATABASE_VERSION = 8 type VersionMismatchError struct { @@ -60,6 +60,7 @@ var MIGRATIONS = []string{ alter table videos add column view_count integer not null default 0`, `alter table users add column is_banned boolean default 0`, `alter table urls add column short_text text not null default ""`, +`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`, } /** diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index 9d47d12..0872cb7 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -302,6 +302,7 @@ func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, ma // Add Bearer token and guest token func ApiRequestAddTokens(req *http.Request) error { req.Header.Set("Authorization", "Bearer " + BEARER_TOKEN) + req.Header.Set("x-twitter-client-language", "en") guestToken, err := GetGuestToken() if err != nil { diff --git a/scraper/api_types.go b/scraper/api_types.go index 6c12332..bbe4893 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -332,6 +332,7 @@ var tombstone_types = map[string]string{ "This Tweet is unavailable. Learn more": "unavailable", "This Tweet violated the Twitter Rules. Learn more": "violated", "This Tweet is from an account that no longer exists. Learn more": "no longer exists", + "Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter. Learn more": "age-restricted", } /** * Insert tweets into GlobalObjects for each tombstone. Returns a list of users that need to