From 6816dcbe2fe5b66ddd24a9eb405d2ce0cb1376e5 Mon Sep 17 00:00:00 2001 From: Jaeger Aquila Date: Thu, 8 Dec 2022 21:44:02 -0500 Subject: [PATCH] generic http implem and fixing package errors --- go.mod | 1 + go.sum | 3 +- scraper/api_request_utils.go | 82 +++++++++++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index a2381b0..29b0a0e 100644 --- a/go.mod +++ b/go.mod @@ -9,5 +9,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.7 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 + golang.org/x/sys v0.3.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 9579380..36c103f 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index c76d7ed..f23dfc8 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -21,27 +21,89 @@ type API struct { } func NewGuestSession() API { - // TODO api-refactor: Return a new API with its GuestToken set using `GetGuestToken()` from "scraper/guest_token.go" - panic("TODO") + // test to check if a guest token is created? Use the existing one? + // test to check if a the api returns the guest token properly + guestAPIString, err := GetGuestToken() + if err != nil { + panic(err) + } + + return API{ + IsAuthenticated: false, + GuestToken: guestAPIString, + AuthenticationToken: "", + } } func (api *API) LogIn(username string, password string) { // TODO authentication: Log in and save the authentication token(s), set `IsAuthenticated = true` + + api.IsAuthenticated = true + panic("TODO") } -func (api API) do_http(url string, cursor string, result *interface{}) { +func (api API) do_http(url string, cursor string, result *interface{}) error { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return fmt.Errorf("Error initializing HTTP request:\n %w", err) + } + + if cursor != "" { + query := req.URL.Query() + query.Add("cursor", cursor) + // TODO Add 'referrer=tweet' to the url passed by the necessesary caller functions + // if is_tweet { + // query.Add("referrer", "tweet") + // } + req.URL.RawQuery = query.Encode() + } + + // TODO Add the query params to the caller functions url + // GetTweet, Search, and GetFeed use it, GetUser and GetSpace do not use. + // ApiRequestAddAllParams(req) + if api.IsAuthenticated { // TODO authentication: add authentication headers/params } else { - // TODO api-refactor: add guest headers / params + req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN) + req.Header.Set("x-twitter-client-language", "en") + req.Header.Set("X-Guest-Token", api.GuestToken) } - // TODO api-refactor: do the HTTP request and unmarshal the result into the `result` struct - // - if `cursor != ""`, then add the cursor to the request as in `UpdateQueryCursor` before - // executing the request. - // - ignore `referrer=tweet` (aka the boolean param) for now - // - ignore retries for now + // TODO retries + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Error executing HTTP request:\n %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 && resp.StatusCode != 403 { + content, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) //If this fails something royally fucked up + } + + responseHeaders := "" + for header := range resp.Header { + responseHeaders += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header)) + } + return fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, responseHeaders, content) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("Error reading response body:\n %w", err) + } + log.Debug(string(body)) + + err = json.Unmarshal(body, result) + if err != nil { + return fmt.Errorf("Error parsing API response:\n %w", err) + } + return nil + } func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) { @@ -137,7 +199,7 @@ func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, mi func (api API) GetSpace(id SpaceID) (SpaceResponse, error) { client := &http.Client{Timeout: 10 * time.Second} - log.Debug("asdfasd") + req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%22"+string(id)+"%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D", //nolint:lll // It's a URL, come on nil) if err != nil {