me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BUGFIX: fix CSRF tokens getting out of sync on requests that load multiple pages of results

Browse Source
This commit is contained in:
Alessio 2023-03-09 22:49:55 -05:00
parent ae9a9c3a47
commit 1384aa73e9
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scraper/api_request_utils.go
View File

@ -188,7 +188,12 @@ func (api *API) LogIn(username string, password string) {
api.UserHandle = UserHandle(final_result.Subtasks[0].OpenAccount.User.ScreenName) api.UserHandle = UserHandle(final_result.Subtasks[0].OpenAccount.User.ScreenName)
dummyURL, err := url.Parse(loginURL) api.update_csrf_token()
api.IsAuthenticated = true
}
func (api *API) update_csrf_token() {
dummyURL, err := url.Parse("https://twitter.com/i/api/1.1/onboarding/task.json")
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -196,16 +201,12 @@ func (api *API) LogIn(username string, password string) {
for _, cookie := range api.Client.Jar.Cookies(dummyURL) { for _, cookie := range api.Client.Jar.Cookies(dummyURL) {
if cookie.Name == "ct0" { if cookie.Name == "ct0" {
api.CSRFToken = cookie.Value api.CSRFToken = cookie.Value
return
} }
} }
if api.CSRFToken == "" { panic("No CSRF Token Found")
panic("No CSRF Token Found")
}
api.IsAuthenticated = true
} }
func (api *API) do_http_POST(url string, body string, result interface{}) error { func (api *API) do_http_POST(url string, body string, result interface{}) error {
req, err := http.NewRequest("POST", url, strings.NewReader(body)) req, err := http.NewRequest("POST", url, strings.NewReader(body))
if err != nil { if err != nil {
@ -248,7 +249,7 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
return nil return nil
} }
func (api API) do_http(url string, cursor string, result interface{}) error { func (api *API) do_http(url string, cursor string, result interface{}) error {
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
return fmt.Errorf("Error initializing HTTP GET request:\n %w", err) return fmt.Errorf("Error initializing HTTP GET request:\n %w", err)
@ -291,6 +292,11 @@ func (api API) do_http(url string, cursor string, result interface{}) error {
if err != nil { if err != nil {
return fmt.Errorf("Error parsing API response:\n %w", err) return fmt.Errorf("Error parsing API response:\n %w", err)
} }
if api.IsAuthenticated {
// New request has been made, so the cookie will be changed; update the csrf to match
api.update_csrf_token()
}
return nil return nil
} }

4
scraper/api_types_v2.go
View File

@ -409,7 +409,7 @@ func get_graphql_user_timeline_url(user_id UserID, cursor string) string {
/** /**
* Get a User feed using the new GraphQL twitter api * Get a User feed using the new GraphQL twitter api
*/ */
func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) { func (api *API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) {
url, err := url.Parse(get_graphql_user_timeline_url(user_id, cursor)) url, err := url.Parse(get_graphql_user_timeline_url(user_id, cursor))
if err != nil { if err != nil {
panic(err) panic(err)
@ -435,7 +435,7 @@ func (api API) GetLikesFor(user_id UserID, cursor string) (APIV2Response, error)
* - response: an "out" parameter; the APIV2Response that tweets, RTs and users will be appended to * - response: an "out" parameter; the APIV2Response that tweets, RTs and users will be appended to
* - min_tweets: the desired minimum amount of tweets to get * - min_tweets: the desired minimum amount of tweets to get
*/ */
func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Response, min_tweets int) error { func (api *API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Response, min_tweets int) error {
// TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop // TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop
// forever getting no new tweets // forever getting no new tweets
last_response := response last_response := response