Update API call functions to use do_http

- move query-string modifications to individual API call functions:
"referrer=tweet", tweet "include_XYZ" params
- tidy up implementation of retries in `API.GetUser`
- fix out-param of `do_http` being a pointer to an interface when it should just be an interface

scraper/api_request_utils.go

@@ -37,13 +37,10 @@ func NewGuestSession() API {
 
 func (api *API) LogIn(username string, password string) {
 	// TODO authentication: Log in and save the authentication token(s), set `IsAuthenticated = true`
-
-	api.IsAuthenticated = true
-
 	panic("TODO")
 }
 
-func (api API) do_http(url string, cursor string, result *interface{}) error {
+func (api API) do_http(url string, cursor string, result interface{}) error {
 	client := &http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
@@ -53,26 +50,23 @@ func (api API) do_http(url string, cursor string, result *interface{}) error {
 	if cursor != "" {
 		query := req.URL.Query()
 		query.Add("cursor", cursor)
-		// TODO Add 'referrer=tweet' to the url passed by the necessesary caller functions
-		// if is_tweet {
-		// 	query.Add("referrer", "tweet")
-		// }
 		req.URL.RawQuery = query.Encode()
 	}
 
-	// TODO Add the query params to the caller functions url
+	// Params for every request
-	// GetTweet, Search, and GetFeed use it, GetUser and GetSpace do not use.
+	req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
-	// ApiRequestAddAllParams(req)
+	req.Header.Set("x-twitter-client-language", "en")
 
 	if api.IsAuthenticated {
 		// TODO authentication: add authentication headers/params
 	} else {
-		req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
+		// Not authenticated; use guest token
-		req.Header.Set("x-twitter-client-language", "en")
+		if api.GuestToken == "" {
+			panic("No guest token set!")
+		}
 		req.Header.Set("X-Guest-Token", api.GuestToken)
 	}
 
-	// TODO retries
 	resp, err := client.Do(req)
 	if err != nil {
 		return fmt.Errorf("Error executing HTTP request:\n  %w", err)
@@ -82,7 +76,7 @@ func (api API) do_http(url string, cursor string, result *interface{}) error {
 	if resp.StatusCode != 200 && resp.StatusCode != 403 {
 		content, err := io.ReadAll(resp.Body)
 		if err != nil {
-			panic(err) //If this fails something royally fucked up
+			panic(err)
 		}
 
 		responseHeaders := ""
@@ -105,54 +99,47 @@ func (api API) do_http(url string, cursor string, result *interface{}) error {
 	return nil
 }
 
+// Add the query params to get all data
+func add_tweet_query_params(query *url.Values) {
+	query.Add("include_profile_interstitial_type", "1")
+	query.Add("include_blocking", "1")
+	query.Add("include_blocked_by", "1")
+	query.Add("include_followed_by", "1")
+	query.Add("include_want_retweets", "1")
+	query.Add("include_mute_edge", "1")
+	query.Add("include_can_dm", "1")
+	query.Add("include_can_media_tag", "1")
+	query.Add("skip_status", "1")
+	query.Add("cards_platform", "Web-12")
+	query.Add("include_cards", "1")
+	query.Add("include_ext_alt_text", "true")
+	query.Add("include_quote_count", "true")
+	query.Add("include_reply_count", "1")
+	query.Add("tweet_mode", "extended")
+	query.Add("include_entities", "true")
+	query.Add("include_user_entities", "true")
+	query.Add("include_ext_media_availability", "true")
+	query.Add("send_error_codes", "true")
+	query.Add("simple_quoted_tweet", "true")
+	query.Add("include_tweet_replies", "true")
+	query.Add("ext", "mediaStats,highlightedLabel")
+	query.Add("count", "20")
+}
+
 func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	// TODO: this function isn't actually used for anything (APIv2 is used instead)
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_USER_TIMELINE_BASE_PATH, user_id), nil)
+	url, err := url.Parse(fmt.Sprintf("%s%d.json", API_USER_TIMELINE_BASE_PATH, user_id))
 	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error initializing HTTP request for GetFeedFor(%d):\n  %w", user_id, err)
+		panic(err)
 	}
+	queryParams := url.Query()
+	add_tweet_query_params(&queryParams)
+	url.RawQuery = queryParams.Encode()
 
-	err = ApiRequestAddTokens(req)
+	var result TweetResponse
-	if err != nil {
+	err = api.do_http(url.String(), cursor, &result)
-		return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
-	}
 
-	ApiRequestAddAllParams(req)
+	return result, err
-
-	if cursor != "" {
-		UpdateQueryCursor(req, cursor, false)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error executing HTTP request for GetFeedFor(%d):\n  %w", user_id, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		content, err := io.ReadAll(resp.Body)
-		if err != nil {
-			panic(err)
-		}
-		s := ""
-		for header := range resp.Header {
-			s += fmt.Sprintf("    %s: %s\n", header, resp.Header.Get(header))
-		}
-		return TweetResponse{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error reading response body for GetUserFeedFor(%d):\n  %w", user_id, err)
-	}
-	log.Debug(string(body))
-
-	var response TweetResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return response, fmt.Errorf("Error parsing API response for GetUserFeedFor(%d):\n  %w", user_id, err)
-	}
-	return response, nil
 }
 
 /**
@@ -197,87 +184,32 @@ func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, mi
 }
 
 func (api API) GetSpace(id SpaceID) (SpaceResponse, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	// TODO: break up this URL into params so it's readable
-
+	url, err := url.Parse("https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%22" + string(id) + "%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D") //nolint:lll  // It's a URL, come on
-	req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%22"+string(id)+"%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D", //nolint:lll  // It's a URL, come on
-		nil)
 	if err != nil {
-		return SpaceResponse{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+		panic(err)
-	}
-	err = ApiRequestAddTokens(req)
-	if err != nil {
-		return SpaceResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
 	}
 
-	resp, err := client.Do(req)
+	var result SpaceResponse
-	if err != nil {
+	err = api.do_http(url.String(), "", &result)
-		return SpaceResponse{}, fmt.Errorf("Error executing HTTP request for GetSpace(%s):\n  %w", id, err)
+	return result, err
-	}
-	defer resp.Body.Close()
-	if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) {
-		content, err := io.ReadAll(resp.Body)
-		if err != nil {
-			panic(err)
-		}
-		return SpaceResponse{}, fmt.Errorf("Error getting %q.  HTTP %s: %s", req.URL, resp.Status, content)
-	}
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return SpaceResponse{}, fmt.Errorf("Error reading HTTP request:\n  %w", err)
-	}
-	log.Debug(string(body))
-
-	var response SpaceResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return response, fmt.Errorf("Error parsing API response for GetSpace(%s):\n  %w", id, err)
-	}
-	return response, nil
 }
 
 func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	url, err := url.Parse(fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id))
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id), nil)
 	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+		panic(err)
 	}
-
+	queryParams := url.Query()
-	err = ApiRequestAddTokens(req)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
-	}
-
-	ApiRequestAddAllParams(req)
 	if cursor != "" {
-		UpdateQueryCursor(req, cursor, true)
+		queryParams.Add("referrer", "tweet")
 	}
+	add_tweet_query_params(&queryParams)
+	url.RawQuery = queryParams.Encode()
 
-	resp, err := client.Do(req)
+	var result TweetResponse
-	if err != nil {
+	err = api.do_http(url.String(), cursor, &result)
-		return TweetResponse{}, fmt.Errorf("Error executing HTTP request:\n  %w", err)
+	return result, err
-	}
-	defer resp.Body.Close()
-
-	if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) {
-		content, err := io.ReadAll(resp.Body)
-		if err != nil {
-			panic(err)
-		}
-		return TweetResponse{}, fmt.Errorf("Error getting %q.  HTTP %s: %s", req.URL, resp.Status, content)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error reading HTTP request:\n  %w", err)
-	}
-	log.Debug(string(body))
-
-	var response TweetResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return response, fmt.Errorf("Error parsing API response for GetTweet(%d):\n  %w", id, err)
-	}
-	return response, nil
 }
 
 // Resend the request to get more replies if necessary
@@ -312,111 +244,55 @@ func UpdateQueryCursor(req *http.Request, new_cursor string, is_tweet bool) {
 }
 
 func (api API) GetUser(handle UserHandle) (APIUser, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	// TODO: break up this URL into params so it's readable
-	req, err := http.NewRequest(
+	url, err := url.Parse("https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22" + string(handle) +
-		"GET",
+		"%22%2C%22withHighlightedLabel%22%3Atrue%7D")
-		"https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22"+string(handle)+
-			"%22%2C%22withHighlightedLabel%22%3Atrue%7D",
-		nil)
 	if err != nil {
-		return APIUser{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+		panic(err)
-	}
-	err = ApiRequestAddTokens(req)
-	if err != nil {
-		return APIUser{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
 	}
 
-	var response UserResponse
+	var result UserResponse
 	for retries := 0; retries < 3; retries += 1 {
-		resp, err := client.Do(req)
+		result = UserResponse{} // Clear any previous result
+		err = api.do_http(url.String(), "", &result)
 		if err != nil {
-			return APIUser{}, fmt.Errorf("Error executing HTTP request for GetUser(%s):\n  %w", handle, err)
+			return APIUser{}, err
-		}
-		defer resp.Body.Close()
-
-		// Sometimes it randomly gives 403 Forbidden.  API's fault, not ours
-		// We check for this below
-		if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) {
-			content, err := io.ReadAll(resp.Body)
-			if err != nil {
-				panic(err)
-			}
-			return APIUser{}, fmt.Errorf("response status %s: %s", resp.Status, content)
 		}
 
-		body, err := io.ReadAll(resp.Body)
+		if len(result.Errors) == 0 {
-		if err != nil {
+			// Success; no retrying needed
-			return APIUser{}, fmt.Errorf("Error retrieving API response to GetUser(%s):\n  %w", handle, err)
+			break
-		}
-		log.Debug("GetUser(" + string(handle) + "): " + string(body))
-
-		err = json.Unmarshal(body, &response)
-		if err != nil {
-			return APIUser{}, fmt.Errorf("Error parsing API response to GetUser(%s):\n  %w", handle, err)
 		}
 
-		// Retry ONLY if the error is code 50 (random authentication failure), NOT on real errors
+		if result.Errors[0].Code != 50 || result.Errors[0].Name == "NotFoundError" {
-		if len(response.Errors) == 1 && response.Errors[0].Code == 50 && response.Errors[0].Name != "NotFoundError" {
+			// Retry ONLY if the error is code 50 (random authentication failure)
-			// Reset the response (remove the Errors)
+			// Do NOT retry on real errors
-			response = UserResponse{}
-			continue
-		} else {
-			// Do not retry on real errors
 			break
 		}
 	}
-	return response.ConvertToAPIUser(), err
+
+	return result.ConvertToAPIUser(), err
 }
 
 func (api API) Search(query string, cursor string) (TweetResponse, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	url, err := url.Parse("https://twitter.com/i/api/2/search/adaptive.json")
-	req, err := http.NewRequest(
-		"GET",
-		"https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q="+
-			url.QueryEscape(query),
-		nil)
 	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+		panic(err)
 	}
 
-	err = ApiRequestAddTokens(req)
+	queryParams := url.Query()
-	if err != nil {
+	add_tweet_query_params(&queryParams)
-		return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
+	queryParams.Add("count", "50")
-	}
+	queryParams.Add("spelling_corrections", "1")
+	queryParams.Add("query_source", "typed_query")
+	queryParams.Add("pc", "1")
+	queryParams.Add("q", query)
+	url.RawQuery = queryParams.Encode()
+	fmt.Println(url.RawQuery)
 
-	ApiRequestAddAllParams(req)
+	var result TweetResponse
-	if cursor != "" {
+	err = api.do_http(url.String(), cursor, &result)
-		UpdateQueryCursor(req, cursor, false)
+	return result, err
-	}
-
-	fmt.Println(req.URL.String())
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error executing HTTP request for Search(%q):\n  %w", query, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		content, err := io.ReadAll(resp.Body)
-		if err != nil {
-			panic(err)
-		}
-		return TweetResponse{}, fmt.Errorf("Error while searching for %q.  HTTP %s: %s", req.URL, resp.Status, content)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return TweetResponse{}, fmt.Errorf("Error retrieving API response for Search(%q):\n  %w", query, err)
-	}
-	// fmt.Println(string(body))
-
-	var response TweetResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return response, fmt.Errorf("Error parsing API response to Search(%q):\n  %w", query, err)
-	}
-	return response, nil
 }
 
 func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, max_results int) error {
@@ -445,45 +321,3 @@ func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, ma
 	}
 	return nil
 }
-
-// Add Bearer token and guest token
-func ApiRequestAddTokens(req *http.Request) error {
-	req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
-	req.Header.Set("x-twitter-client-language", "en")
-
-	guestToken, err := GetGuestToken()
-	if err != nil {
-		return err
-	}
-	req.Header.Set("X-Guest-Token", guestToken)
-	return nil
-}
-
-// Add the query params to get all data
-func ApiRequestAddAllParams(req *http.Request) {
-	query := req.URL.Query()
-	query.Add("include_profile_interstitial_type", "1")
-	query.Add("include_blocking", "1")
-	query.Add("include_blocked_by", "1")
-	query.Add("include_followed_by", "1")
-	query.Add("include_want_retweets", "1")
-	query.Add("include_mute_edge", "1")
-	query.Add("include_can_dm", "1")
-	query.Add("include_can_media_tag", "1")
-	query.Add("skip_status", "1")
-	query.Add("cards_platform", "Web-12")
-	query.Add("include_cards", "1")
-	query.Add("include_ext_alt_text", "true")
-	query.Add("include_quote_count", "true")
-	query.Add("include_reply_count", "1")
-	query.Add("tweet_mode", "extended")
-	query.Add("include_entities", "true")
-	query.Add("include_user_entities", "true")
-	query.Add("include_ext_media_availability", "true")
-	query.Add("send_error_codes", "true")
-	query.Add("simple_quoted_tweet", "true")
-	query.Add("include_tweet_replies", "true")
-	query.Add("ext", "mediaStats,highlightedLabel")
-	query.Add("count", "20")
-	req.URL.RawQuery = query.Encode()
-}

scraper/api_types_v2.go

@@ -420,10 +420,14 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response,
 		return APIV2Response{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
 	}
 
-	err = ApiRequestAddTokens(req)
+	req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
+	req.Header.Set("x-twitter-client-language", "en")
+
+	guestToken, err := GetGuestToken()
 	if err != nil {
 		return APIV2Response{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
 	}
+	req.Header.Set("X-Guest-Token", guestToken)
 
 	if cursor != "" {
 		UpdateQueryCursor(req, cursor, false)