diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index a3b46cb..85dbb39 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -21,8 +21,6 @@ type API struct { } func NewGuestSession() API { - // test to check if a guest token is created? Use the existing one? - // test to check if a the api returns the guest token properly guestAPIString, err := GetGuestToken() if err != nil { panic(err) diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 87b63ee..35392b0 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -1,13 +1,9 @@ package scraper import ( - "encoding/json" "fmt" - "io" - "net/http" "net/url" "strings" - "time" log "github.com/sirupsen/logrus" ) @@ -414,57 +410,15 @@ func get_graphql_user_timeline_url(user_id UserID, cursor string) string { * Get a User feed using the new GraphQL twitter api */ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, error) { - client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest("GET", get_graphql_user_timeline_url(user_id, cursor), nil) + url, err := url.Parse(get_graphql_user_timeline_url(user_id, cursor)) if err != nil { - return APIV2Response{}, fmt.Errorf("Error initializing HTTP request:\n %w", err) + panic(err) } - req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN) - req.Header.Set("x-twitter-client-language", "en") - - guestToken, err := GetGuestToken() - if err != nil { - return APIV2Response{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err) - } - req.Header.Set("X-Guest-Token", guestToken) - - if cursor != "" { - query := req.URL.Query() - query.Add("cursor", cursor) - req.URL.RawQuery = query.Encode() - } - - resp, err := client.Do(req) - if err != nil { - return APIV2Response{}, fmt.Errorf("Error executing HTTP request:\n %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - content, err := io.ReadAll(resp.Body) - if err != nil { - panic(err) - } - s := "" - for header := range resp.Header { - s += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header)) - } - return APIV2Response{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return APIV2Response{}, fmt.Errorf("Error reading HTTP response body:\n %w", err) - } - log.Debug(string(body)) - var response APIV2Response - err = json.Unmarshal(body, &response) - if err != nil { - return response, fmt.Errorf("Error parsing API response for GetGraphqlFeedFor(%d):\n %w", user_id, err) - } - return response, nil + err = api.do_http(url.String(), cursor, &response) + + return response, err } /** diff --git a/scraper/user_feed.go b/scraper/user_feed.go index ce1305e..e92880c 100644 --- a/scraper/user_feed.go +++ b/scraper/user_feed.go @@ -16,7 +16,7 @@ import ( * returns: a slice of Tweets, Retweets, and Users */ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) { - api := API{} + api := NewGuestSession() tweet_response, err := api.GetFeedFor(user_id, "") if err != nil { err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err) @@ -34,7 +34,7 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error } func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) { - api := API{} + api := NewGuestSession() api_response, err := api.GetGraphqlFeedFor(user_id, "") if err != nil { err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err)