Finished refactoring

scraper/api_request_utils.go

@@ -21,8 +21,6 @@ type API struct {
 }
 
 func NewGuestSession() API {
-	// test to check if a guest token is created? Use the existing one?
-	// test to check if a the api returns the guest token properly
 	guestAPIString, err := GetGuestToken()
 	if err != nil {
 		panic(err)

scraper/api_types_v2.go

@@ -1,13 +1,9 @@
 package scraper
 
 import (
-	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -414,57 +410,15 @@ func get_graphql_user_timeline_url(user_id UserID, cursor string) string {
  * Get a User feed using the new GraphQL twitter api
  */
 func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) {
-	client := &http.Client{Timeout: 10 * time.Second}
+	url, err := url.Parse(get_graphql_user_timeline_url(user_id, cursor))
-	req, err := http.NewRequest("GET", get_graphql_user_timeline_url(user_id, cursor), nil)
-	if err != nil {
-		return APIV2Response{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
-	}
-
-	req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
-	req.Header.Set("x-twitter-client-language", "en")
-
-	guestToken, err := GetGuestToken()
-	if err != nil {
-		return APIV2Response{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
-	}
-	req.Header.Set("X-Guest-Token", guestToken)
-
-	if cursor != "" {
-		query := req.URL.Query()
-		query.Add("cursor", cursor)
-		req.URL.RawQuery = query.Encode()
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return APIV2Response{}, fmt.Errorf("Error executing HTTP request:\n  %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		content, err := io.ReadAll(resp.Body)
 	if err != nil {
 		panic(err)
 	}
-		s := ""
-		for header := range resp.Header {
-			s += fmt.Sprintf("    %s: %s\n", header, resp.Header.Get(header))
-		}
-		return APIV2Response{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return APIV2Response{}, fmt.Errorf("Error reading HTTP response body:\n  %w", err)
-	}
-	log.Debug(string(body))
 
 	var response APIV2Response
-	err = json.Unmarshal(body, &response)
+	err = api.do_http(url.String(), cursor, &response)
-	if err != nil {
+
-		return response, fmt.Errorf("Error parsing API response for GetGraphqlFeedFor(%d):\n  %w", user_id, err)
+	return response, err
-	}
-	return response, nil
 }
 
 /**

scraper/user_feed.go

@@ -16,7 +16,7 @@ import (
  * returns: a slice of Tweets, Retweets, and Users
  */
 func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) {
-	api := API{}
+	api := NewGuestSession()
 	tweet_response, err := api.GetFeedFor(user_id, "")
 	if err != nil {
 		err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n  %w", user_id, err)
@@ -34,7 +34,7 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error
 }
 
 func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) {
-	api := API{}
+	api := NewGuestSession()
 	api_response, err := api.GetGraphqlFeedFor(user_id, "")
 	if err != nil {
 		err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n  %w", user_id, err)