diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index c85ca70..3ef4b48 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -188,7 +188,12 @@ func (api *API) LogIn(username string, password string) { api.UserHandle = UserHandle(final_result.Subtasks[0].OpenAccount.User.ScreenName) - dummyURL, err := url.Parse(loginURL) + api.update_csrf_token() + api.IsAuthenticated = true +} + +func (api *API) update_csrf_token() { + dummyURL, err := url.Parse("https://twitter.com/i/api/1.1/onboarding/task.json") if err != nil { panic(err) } @@ -196,16 +201,12 @@ func (api *API) LogIn(username string, password string) { for _, cookie := range api.Client.Jar.Cookies(dummyURL) { if cookie.Name == "ct0" { api.CSRFToken = cookie.Value + return } } - if api.CSRFToken == "" { - panic("No CSRF Token Found") - } - - api.IsAuthenticated = true + panic("No CSRF Token Found") } - func (api *API) do_http_POST(url string, body string, result interface{}) error { req, err := http.NewRequest("POST", url, strings.NewReader(body)) if err != nil { @@ -248,7 +249,7 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error return nil } -func (api API) do_http(url string, cursor string, result interface{}) error { +func (api *API) do_http(url string, cursor string, result interface{}) error { req, err := http.NewRequest("GET", url, nil) if err != nil { return fmt.Errorf("Error initializing HTTP GET request:\n %w", err) @@ -291,6 +292,11 @@ func (api API) do_http(url string, cursor string, result interface{}) error { if err != nil { return fmt.Errorf("Error parsing API response:\n %w", err) } + + if api.IsAuthenticated { + // New request has been made, so the cookie will be changed; update the csrf to match + api.update_csrf_token() + } return nil } diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index ea3701a..835d111 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -409,7 +409,7 @@ func get_graphql_user_timeline_url(user_id UserID, cursor string) string { /** * Get a User feed using the new GraphQL twitter api */ -func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) { +func (api *API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) { url, err := url.Parse(get_graphql_user_timeline_url(user_id, cursor)) if err != nil { panic(err) @@ -435,7 +435,7 @@ func (api API) GetLikesFor(user_id UserID, cursor string) (APIV2Response, error) * - response: an "out" parameter; the APIV2Response that tweets, RTs and users will be appended to * - min_tweets: the desired minimum amount of tweets to get */ -func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Response, min_tweets int) error { +func (api *API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Response, min_tweets int) error { // TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop // forever getting no new tweets last_response := response