Enable fetching of Space details

scraper/api_request_utils.go

@@ -107,6 +107,45 @@ func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, mi
 	return nil
 }
 
+func (api API) GetSpace(id SpaceID) (SpaceResponse, error) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	log.Debug("asdfasd")
+	req, err := http.NewRequest("GET", "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%22"+string(id)+"%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D", //nolint:lll  // It's a URL, come on
+		nil)
+	if err != nil {
+		return SpaceResponse{}, fmt.Errorf("Error initializing HTTP request:\n  %w", err)
+	}
+	err = ApiRequestAddTokens(req)
+	if err != nil {
+		return SpaceResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n  %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return SpaceResponse{}, fmt.Errorf("Error executing HTTP request for GetSpace(%s):\n  %w", id, err)
+	}
+	defer resp.Body.Close()
+	if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) {
+		content, err := io.ReadAll(resp.Body)
+		if err != nil {
+			panic(err)
+		}
+		return SpaceResponse{}, fmt.Errorf("Error getting %q.  HTTP %s: %s", req.URL, resp.Status, content)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return SpaceResponse{}, fmt.Errorf("Error reading HTTP request:\n  %w", err)
+	}
+	log.Debug(string(body))
+
+	var response SpaceResponse
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return response, fmt.Errorf("Error parsing API response for GetSpace(%s):\n  %w", id, err)
+	}
+	return response, nil
+}
+
 func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
 	client := &http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id), nil)

scraper/space.go

@@ -1,5 +1,9 @@
 package scraper
 
+import (
+	"fmt"
+)
+
 type SpaceID string
 
 type Space struct {
@@ -32,3 +36,12 @@ func ParseAPISpace(apiCard APICard) Space {
 
 	return ret
 }
+
+func FetchSpaceDetail(id SpaceID) (TweetTrove, error) {
+	api := API{}
+	space_response, err := api.GetSpace(id)
+	if err != nil {
+		return TweetTrove{}, fmt.Errorf("Error in API call to fetch Space (id %q):\n  %w", id, err)
+	}
+	return space_response.ToTweetTrove(), nil
+}

scraper/tweet_trove.go

@@ -120,8 +120,31 @@ func (trove *TweetTrove) FillMissingUserIDs() {
 	}
 }
 
+func (trove *TweetTrove) FillSpaceDetails() error {
+	fmt.Println("Filling space details")
+	for i := range trove.Spaces {
+		fmt.Printf("Getting space: %q\n", trove.Spaces[i].ID)
+		new_trove, err := FetchSpaceDetail(trove.Spaces[i].ID)
+		if err != nil {
+			return err
+		}
+		// Replace the old space in the trove with the new, updated one
+		new_space, is_ok := new_trove.Spaces[i]
+		if is_ok {
+			// Necessary to check is_ok because the space response could be empty, in which case
+			// we don't want to overwrite it
+			trove.Spaces[i] = new_space
+		}
+	}
+	return nil
+}
+
 func (trove *TweetTrove) PostProcess() error {
 	trove.FetchTombstoneUsers()
 	trove.FillMissingUserIDs()
+	err := trove.FillSpaceDetails()
+	if err != nil {
+		return err
+	}
 	return nil
 }