me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do slightly more proper error handling

Browse Source
This commit is contained in:
Alessio 2022-03-06 19:27:30 -08:00
parent 2f784c779c
commit 15d7cd77a1
6 changed files with 22 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scraper/api_errors.go Normal file
View File

@ -0,0 +1,8 @@
package scraper
import (
"fmt"
)
var END_OF_FEED = fmt.Errorf("End of feed")
var DOESNT_EXIST = fmt.Errorf("Doesn't exist")

7
scraper/api_request_utils.go
View File

@ -14,13 +14,6 @@ import (
const API_CONVERSATION_BASE_PATH = "https://twitter.com/i/api/2/timeline/conversation/"
const API_USER_TIMELINE_BASE_PATH = "https://api.twitter.com/2/timeline/profile/"
type APIError string
func (e APIError) Error() string {
return string(e)
}
const END_OF_FEED = APIError("End of feed")
type API struct{}
func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) {

9
scraper/search.go
View File

@ -1,5 +1,8 @@
package scraper
import (
"errors"
)
func TimestampToDateString(timestamp int) string {
panic("???") // TODO
@ -22,14 +25,12 @@ func Search(query string, min_results int) (trove TweetTrove, err error) {
if len(tweet_response.GlobalObjects.Tweets) < min_results && tweet_response.GetCursor() != "" {
err = api.GetMoreTweetsFromSearch(query, &tweet_response, min_results)
if err == END_OF_FEED {
if errors.Is(err, END_OF_FEED) {
println("End of feed!")
}
if err != nil && err != END_OF_FEED {
} else if err != nil {
return
}
}
return ParseTweetResponse(tweet_response)
}

4
scraper/tweet.go
View File

@ -93,7 +93,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
if apiTweet.TombstoneText == "" { // Skip time parsing for tombstones
ret.PostedAt, err = TimestampFromString(apiTweet.CreatedAt)
if err != nil {
return
return Tweet{}, fmt.Errorf("Error parsing time on tweet ID %d:\n %w", ret.ID, err)
}
}
@ -223,12 +223,14 @@ func GetTweetFull(id TweetID) (trove TweetTrove, err error) {
api := API{}
tweet_response, err := api.GetTweet(id, "")
if err != nil {
err = fmt.Errorf("Error getting tweet: %d\n %w", id, err)
return
}
if len(tweet_response.GlobalObjects.Tweets) < DEFAULT_MAX_REPLIES_EAGER_LOAD &&
tweet_response.GetCursor() != "" {
err = api.GetMoreReplies(id, &tweet_response, DEFAULT_MAX_REPLIES_EAGER_LOAD)
if err != nil {
err = fmt.Errorf("Error getting more tweet replies: %d\n %w", id, err)
return
}
}

1
scraper/user.go
View File

@ -134,6 +134,7 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) {
}
ret.JoinDate, err = TimestampFromString(apiUser.CreatedAt)
if err != nil {
err = fmt.Errorf("Error parsing time on user ID %d: %w", ret.ID, err)
return
}
ret.IsPrivate = apiUser.Protected

8
scraper/user_feed.go
View File

@ -2,6 +2,7 @@ package scraper
import (
"fmt"
"errors"
)
/**
@ -18,12 +19,13 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error
api := API{}
tweet_response, err := api.GetFeedFor(user_id, "")
if err != nil {
err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err)
return
}
if len(tweet_response.GlobalObjects.Tweets) < min_tweets && tweet_response.GetCursor() != "" {
err = api.GetMoreTweetsFromFeed(user_id, &tweet_response, min_tweets)
if err != nil && err != END_OF_FEED {
if err != nil && !errors.Is(err, END_OF_FEED) {
return
}
}
@ -36,13 +38,13 @@ func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, er
api := API{}
api_response, err := api.GetGraphqlFeedFor(user_id, "")
if err != nil {
err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %s", user_id, err.Error())
err = fmt.Errorf("Error calling API to fetch user feed: UserID %d\n %w", user_id, err)
return
}
if len(api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) < min_tweets && api_response.GetCursorBottom() != "" {
err = api.GetMoreTweetsFromGraphqlFeed(user_id, &api_response, min_tweets)
if err != nil && err != END_OF_FEED {
if err != nil && !errors.Is(err, END_OF_FEED) {
return
}
}