diff --git a/scraper/api_errors.go b/scraper/api_errors.go new file mode 100644 index 0000000..c257c71 --- /dev/null +++ b/scraper/api_errors.go @@ -0,0 +1,8 @@ +package scraper + +import ( + "fmt" +) + +var END_OF_FEED = fmt.Errorf("End of feed") +var DOESNT_EXIST = fmt.Errorf("Doesn't exist") diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index 5b80cb5..0b8bb9b 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -14,13 +14,6 @@ import ( const API_CONVERSATION_BASE_PATH = "https://twitter.com/i/api/2/timeline/conversation/" const API_USER_TIMELINE_BASE_PATH = "https://api.twitter.com/2/timeline/profile/" -type APIError string -func (e APIError) Error() string { - return string(e) -} - -const END_OF_FEED = APIError("End of feed") - type API struct{} func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) { diff --git a/scraper/search.go b/scraper/search.go index 012ade1..44fe8e1 100644 --- a/scraper/search.go +++ b/scraper/search.go @@ -1,5 +1,8 @@ package scraper +import ( + "errors" +) func TimestampToDateString(timestamp int) string { panic("???") // TODO @@ -22,14 +25,12 @@ func Search(query string, min_results int) (trove TweetTrove, err error) { if len(tweet_response.GlobalObjects.Tweets) < min_results && tweet_response.GetCursor() != "" { err = api.GetMoreTweetsFromSearch(query, &tweet_response, min_results) - if err == END_OF_FEED { + if errors.Is(err, END_OF_FEED) { println("End of feed!") - } - if err != nil && err != END_OF_FEED { + } else if err != nil { return } } - return ParseTweetResponse(tweet_response) } diff --git a/scraper/tweet.go b/scraper/tweet.go index 9c210ec..965e80e 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -93,7 +93,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { if apiTweet.TombstoneText == "" { // Skip time parsing for tombstones ret.PostedAt, err = TimestampFromString(apiTweet.CreatedAt) if err != nil { - return + return Tweet{}, fmt.Errorf("Error parsing time on tweet ID %d:\n %w", ret.ID, err) } } @@ -223,12 +223,14 @@ func GetTweetFull(id TweetID) (trove TweetTrove, err error) { api := API{} tweet_response, err := api.GetTweet(id, "") if err != nil { + err = fmt.Errorf("Error getting tweet: %d\n %w", id, err) return } if len(tweet_response.GlobalObjects.Tweets) < DEFAULT_MAX_REPLIES_EAGER_LOAD && tweet_response.GetCursor() != "" { err = api.GetMoreReplies(id, &tweet_response, DEFAULT_MAX_REPLIES_EAGER_LOAD) if err != nil { + err = fmt.Errorf("Error getting more tweet replies: %d\n %w", id, err) return } } diff --git a/scraper/user.go b/scraper/user.go index 1b7977e..b23beac 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -134,6 +134,7 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) { } ret.JoinDate, err = TimestampFromString(apiUser.CreatedAt) if err != nil { + err = fmt.Errorf("Error parsing time on user ID %d: %w", ret.ID, err) return } ret.IsPrivate = apiUser.Protected diff --git a/scraper/user_feed.go b/scraper/user_feed.go index 2bafd66..aeffdb5 100644 --- a/scraper/user_feed.go +++ b/scraper/user_feed.go @@ -2,6 +2,7 @@ package scraper import ( "fmt" + "errors" ) /** @@ -18,12 +19,13 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error api := API{} tweet_response, err := api.GetFeedFor(user_id, "") if err != nil { + err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err) return } if len(tweet_response.GlobalObjects.Tweets) < min_tweets && tweet_response.GetCursor() != "" { err = api.GetMoreTweetsFromFeed(user_id, &tweet_response, min_tweets) - if err != nil && err != END_OF_FEED { + if err != nil && !errors.Is(err, END_OF_FEED) { return } } @@ -36,13 +38,13 @@ func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, er api := API{} api_response, err := api.GetGraphqlFeedFor(user_id, "") if err != nil { - err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %s", user_id, err.Error()) + err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err) return } if len(api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) < min_tweets && api_response.GetCursorBottom() != "" { err = api.GetMoreTweetsFromGraphqlFeed(user_id, &api_response, min_tweets) - if err != nil && err != END_OF_FEED { + if err != nil && !errors.Is(err, END_OF_FEED) { return } }