diff --git a/scraper/api_types.go b/scraper/api_types.go index ff438f3..0ac37e2 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -458,6 +458,38 @@ func (t *TweetResponse) IsEndOfFeed() bool { return true } +func (t *TweetResponse) ToTweetTrove() (TweetTrove, error) { + ret := NewTweetTrove() + + for _, single_tweet := range t.GlobalObjects.Tweets { + if single_tweet.RetweetedStatusIDStr == "" { + new_tweet, err := ParseSingleTweet(single_tweet) + if err != nil { + return ret, err + } + ret.Tweets[new_tweet.ID] = new_tweet + for _, space := range new_tweet.Spaces { + ret.Spaces[space.ID] = space + } + } else { + new_retweet, err := ParseSingleRetweet(single_tweet) + if err != nil { + return ret, err + } + ret.Retweets[new_retweet.RetweetID] = new_retweet + } + } + + for _, user := range t.GlobalObjects.Users { + new_user, err := ParseSingleUser(user) + if err != nil { + return ret, err + } + ret.Users[new_user.ID] = new_user + } + return ret, nil +} + func idstr_to_int(idstr string) int64 { id, err := strconv.Atoi(idstr) if err != nil { diff --git a/scraper/search.go b/scraper/search.go index e7c3880..597e378 100644 --- a/scraper/search.go +++ b/scraper/search.go @@ -32,7 +32,7 @@ func Search(query string, min_results int) (trove TweetTrove, err error) { } } - trove, err = ParseTweetResponse(tweet_response) + trove, err = tweet_response.ToTweetTrove() if err != nil { err = fmt.Errorf("Error parsing the tweet trove for search query %q:\n %w", query, err) return diff --git a/scraper/tweet.go b/scraper/tweet.go index 2685423..4eadfa6 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -55,9 +55,9 @@ type Tweet struct { ReplyMentions CommaSeparatedList `db:"reply_mentions"` Hashtags CommaSeparatedList `db:"hashtags"` - // TODO get-rid-of-spaces: Might be good to get rid of `Spaces`. Only used in APIv1 I think. + // TODO get-rid-of-redundant-spaces: Might be good to get rid of `Spaces`. Only used in APIv1 I think. // A first-step would be to delete the Spaces after pulling them out of a Tweet into the Trove - // in ParseTweetResponse. Then they will only be getting saved once rather than twice. + // in ToTweetTrove. Then they will only be getting saved once rather than twice. Spaces []Space SpaceID SpaceID `db:"space_id"` @@ -273,10 +273,10 @@ func GetTweetFull(id TweetID, how_many int) (trove TweetTrove, err error) { } } - // This has to be called BEFORE ParseTweetResponse, because it modifies the TweetResponse (adds tombstone tweets to its tweets list) + // This has to be called BEFORE ToTweetTrove, because it modifies the TweetResponse (adds tombstone tweets to its tweets list) tombstoned_users := tweet_response.HandleTombstones() - trove, err = ParseTweetResponse(tweet_response) + trove, err = tweet_response.ToTweetTrove() if err != nil { panic(err) } @@ -301,43 +301,3 @@ func GetTweetFull(id TweetID, how_many int) (trove TweetTrove, err error) { return } - -/** - * Parse an API response object into a list of tweets, retweets and users - * - * args: - * - resp: the response from the API - * - * returns: a list of tweets, retweets and users in that response object - */ -func ParseTweetResponse(resp TweetResponse) (TweetTrove, error) { - trove := NewTweetTrove() - - for _, single_tweet := range resp.GlobalObjects.Tweets { - if single_tweet.RetweetedStatusIDStr == "" { - new_tweet, err := ParseSingleTweet(single_tweet) - if err != nil { - return trove, err - } - trove.Tweets[new_tweet.ID] = new_tweet - for _, space := range new_tweet.Spaces { - trove.Spaces[space.ID] = space - } - } else { - new_retweet, err := ParseSingleRetweet(single_tweet) - if err != nil { - return trove, err - } - trove.Retweets[new_retweet.RetweetID] = new_retweet - } - } - - for _, user := range resp.GlobalObjects.Users { - new_user, err := ParseSingleUser(user) - if err != nil { - return trove, err - } - trove.Users[new_user.ID] = new_user - } - return trove, nil -} diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 0c0237e..af16699 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -227,7 +227,7 @@ func TestParseTweetResponse(t *testing.T) { err = json.Unmarshal(data, &tweet_resp) require.NoError(t, err) - trove, err := ParseTweetResponse(tweet_resp) + trove, err := tweet_resp.ToTweetTrove() require.NoError(t, err) assert.Len(trove.Tweets, 29-3) diff --git a/scraper/user_feed.go b/scraper/user_feed.go index acf1c9c..84d41e8 100644 --- a/scraper/user_feed.go +++ b/scraper/user_feed.go @@ -29,7 +29,7 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error } } - return ParseTweetResponse(tweet_response) + return tweet_response.ToTweetTrove() } func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) {