generic http implem and fixing package errors

go.mod

@@ -9,5 +9,6 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.7
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
+	golang.org/x/sys v0.3.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -22,8 +22,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

scraper/api_request_utils.go

@@ -21,27 +21,89 @@ type API struct {
 }
 
 func NewGuestSession() API {
-	// TODO api-refactor: Return a new API with its GuestToken set using `GetGuestToken()` from "scraper/guest_token.go"
+	// test to check if a guest token is created? Use the existing one?
-	panic("TODO")
+	// test to check if a the api returns the guest token properly
+	guestAPIString, err := GetGuestToken()
+	if err != nil {
+		panic(err)
+	}
+
+	return API{
+		IsAuthenticated:     false,
+		GuestToken:          guestAPIString,
+		AuthenticationToken: "",
+	}
 }
 
 func (api *API) LogIn(username string, password string) {
 	// TODO authentication: Log in and save the authentication token(s), set `IsAuthenticated = true`
+
+	api.IsAuthenticated = true
+
 	panic("TODO")
 }
 
-func (api API) do_http(url string, cursor string, result *interface{}) {
+func (api API) do_http(url string, cursor string, result *interface{}) error {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+	}
+
+	if cursor != "" {
+		query := req.URL.Query()
+		query.Add("cursor", cursor)
+		// TODO Add 'referrer=tweet' to the url passed by the necessesary caller functions
+		// if is_tweet {
+		// 	query.Add("referrer", "tweet")
+		// }
+		req.URL.RawQuery = query.Encode()
+	}
+
+	// TODO Add the query params to the caller functions url
+	// GetTweet, Search, and GetFeed use it, GetUser and GetSpace do not use.
+	// ApiRequestAddAllParams(req)
+
 	if api.IsAuthenticated {
 		// TODO authentication: add authentication headers/params
 	} else {
-		// TODO api-refactor: add guest headers / params
+		req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
+		req.Header.Set("x-twitter-client-language", "en")
+		req.Header.Set("X-Guest-Token", api.GuestToken)
 	}
 
-	// TODO api-refactor: do the HTTP request and unmarshal the result into the `result` struct
+	// TODO retries
-	// - if `cursor != ""`, then add the cursor to the request as in `UpdateQueryCursor` before
+	resp, err := client.Do(req)
-	// executing the request.
+	if err != nil {
-	// - ignore `referrer=tweet` (aka the boolean param) for now
+		return fmt.Errorf("Error executing HTTP request:\n  %w", err)
-	// - ignore retries for now
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 && resp.StatusCode != 403 {
+		content, err := io.ReadAll(resp.Body)
+		if err != nil {
+			panic(err) //If this fails something royally fucked up
+		}
+
+		responseHeaders := ""
+		for header := range resp.Header {
+			responseHeaders += fmt.Sprintf("    %s: %s\n", header, resp.Header.Get(header))
+		}
+		return fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, responseHeaders, content)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("Error reading response body:\n  %w", err)
+	}
+	log.Debug(string(body))
+
+	err = json.Unmarshal(body, result)
+	if err != nil {
+		return fmt.Errorf("Error parsing API response:\n  %w", err)
+	}
+	return nil
+
 }
 
 func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) {
@@ -137,7 +199,7 @@ func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, mi
 
 func (api API) GetSpace(id SpaceID) (SpaceResponse, error) {
 	client := &http.Client{Timeout: 10 * time.Second}
-	log.Debug("asdfasd")
+
 	req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%22"+string(id)+"%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D", //nolint:lll  // It's a URL, come on
 		nil)
 	if err != nil {