me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve closing message when a scrape ends due to rate-limiting

Browse Source
This commit is contained in:
Alessio 2024-06-11 12:32:58 -07:00
parent c454526e47
commit d2045028a7
4 changed files with 24 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/twitter/helpers.go
View File

@ -36,10 +36,10 @@ func die(text string, display_help bool, exit_code int) {
/** /**
* Print a happy exit message and exit * Print a happy exit message and exit
*/ */
func happy_exit(text string, was_rate_limited bool) { func happy_exit(text string, exit_err error) {
if was_rate_limited { if errors.Is(exit_err, scraper.ErrRateLimited) {
fmt.Printf(terminal_utils.COLOR_YELLOW + text + terminal_utils.COLOR_RESET + "\n") fmt.Printf(terminal_utils.COLOR_YELLOW + text + terminal_utils.COLOR_RESET + "\n")
fmt.Printf(terminal_utils.COLOR_YELLOW + "Exiting early (rate limited)." + terminal_utils.COLOR_RESET + "\n") fmt.Printf(terminal_utils.COLOR_YELLOW + "Exiting early (%s)." + terminal_utils.COLOR_RESET + "\n", exit_err.Error())
os.Exit(1) os.Exit(1)
} }
fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n") fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n")

36
cmd/twitter/main.go
View File

@ -237,7 +237,7 @@ func login(username string, password string) {
} }
profile.SaveSession(api) profile.SaveSession(api)
happy_exit("Logged in as "+string(api.UserHandle), false) happy_exit("Logged in as "+string(api.UserHandle), nil)
} }
/** /**
@ -272,7 +272,7 @@ func fetch_user(handle scraper.UserHandle) {
} }
download_user_content(handle) download_user_content(handle)
happy_exit("Saved the user", false) happy_exit("Saved the user", nil)
} }
/** /**
@ -297,7 +297,7 @@ func fetch_tweet_only(tweet_identifier string) {
if err2 != nil { if err2 != nil {
die(fmt.Sprintf("Error saving tweet: %s", err2.Error()), false, 4) die(fmt.Sprintf("Error saving tweet: %s", err2.Error()), false, 4)
} }
happy_exit("Saved the tweet", errors.Is(err, scraper.ErrRateLimited)) happy_exit("Saved the tweet", err)
} }
/** /**
@ -318,7 +318,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
} }
profile.SaveTweetTrove(trove, true) profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), errors.Is(err, scraper.ErrRateLimited)) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
} }
/** /**
@ -341,7 +341,7 @@ func fetch_user_feed(handle string, how_many int) {
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited), err,
) )
} }
@ -359,7 +359,7 @@ func get_user_likes(handle string, how_many int) {
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited), err,
) )
} }
@ -376,7 +376,7 @@ func get_followees(handle string, how_many int) {
profile.SaveTweetTrove(trove, true) profile.SaveTweetTrove(trove, true)
profile.SaveAsFolloweesList(user.ID, trove) profile.SaveAsFolloweesList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), errors.Is(err, scraper.ErrRateLimited)) happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err)
} }
func get_followers(handle string, how_many int) { func get_followers(handle string, how_many int) {
user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) user, err := profile.GetUserByHandle(scraper.UserHandle(handle))
@ -390,7 +390,7 @@ func get_followers(handle string, how_many int) {
profile.SaveTweetTrove(trove, true) profile.SaveTweetTrove(trove, true)
profile.SaveAsFollowersList(user.ID, trove) profile.SaveAsFollowersList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), errors.Is(err, scraper.ErrRateLimited)) happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), err)
} }
func get_bookmarks(how_many int) { func get_bookmarks(how_many int) {
trove, err := scraper.GetBookmarks(how_many) trove, err := scraper.GetBookmarks(how_many)
@ -402,7 +402,7 @@ func get_bookmarks(how_many int) {
happy_exit(fmt.Sprintf( happy_exit(fmt.Sprintf(
"Saved %d tweets, %d retweets, %d users, and %d bookmarks", "Saved %d tweets, %d retweets, %d users, and %d bookmarks",
len(trove.Tweets), len(trove.Retweets), len(trove.Users), len(trove.Bookmarks)), len(trove.Tweets), len(trove.Retweets), len(trove.Users), len(trove.Bookmarks)),
errors.Is(err, scraper.ErrRateLimited), err,
) )
} }
func fetch_timeline(is_following_only bool) { func fetch_timeline(is_following_only bool) {
@ -414,7 +414,7 @@ func fetch_timeline(is_following_only bool) {
happy_exit( happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)), fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited), err,
) )
} }
@ -452,7 +452,7 @@ func search(query string, how_many int) {
} }
profile.SaveTweetTrove(trove, true) profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), errors.Is(err, scraper.ErrRateLimited)) happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
} }
func follow_user(handle string, is_followed bool) { func follow_user(handle string, is_followed bool) {
@ -463,9 +463,9 @@ func follow_user(handle string, is_followed bool) {
profile.SetUserFollowed(&user, is_followed) profile.SetUserFollowed(&user, is_followed)
if is_followed { if is_followed {
happy_exit("Followed user: "+handle, false) happy_exit("Followed user: "+handle, nil)
} else { } else {
happy_exit("Unfollowed user: "+handle, false) happy_exit("Unfollowed user: "+handle, nil)
} }
} }
@ -478,7 +478,7 @@ func unlike_tweet(tweet_identifier string) {
if err != nil { if err != nil {
die(err.Error(), false, -10) die(err.Error(), false, -10)
} }
happy_exit("Unliked the tweet.", false) happy_exit("Unliked the tweet.", nil)
} }
func like_tweet(tweet_identifier string) { func like_tweet(tweet_identifier string) {
@ -494,7 +494,7 @@ func like_tweet(tweet_identifier string) {
if err != nil { if err != nil {
die(err.Error(), false, -1) die(err.Error(), false, -1)
} }
happy_exit("Liked the tweet.", false) happy_exit("Liked the tweet.", nil)
} }
func list_followed() { func list_followed() {
@ -511,7 +511,7 @@ func start_webserver(addr string, should_auto_open bool) {
func fetch_inbox(how_many int) { func fetch_inbox(how_many int) {
trove, _ := scraper.GetInbox(how_many) trove, _ := scraper.GetInbox(how_many)
profile.SaveDMTrove(trove, true) profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), false) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
} }
func fetch_dm(id string, how_many int) { func fetch_dm(id string, how_many int) {
@ -524,7 +524,7 @@ func fetch_dm(id string, how_many int) {
profile.SaveDMTrove(trove, true) profile.SaveDMTrove(trove, true)
happy_exit( happy_exit(
fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)),
errors.Is(err, scraper.ErrRateLimited), err,
) )
} }
@ -536,5 +536,5 @@ func send_dm(room_id string, text string, in_reply_to_id int) {
trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id)) trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id))
profile.SaveDMTrove(trove, true) profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), false) happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
} }

4
pkg/scraper/api_types.go
View File

@ -373,12 +373,12 @@ var tombstone_types = map[string]string{
"Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, " + "Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, " +
"you’ll need to log in to Twitter. Learn more": "age-restricted", "you’ll need to log in to Twitter. Learn more": "age-restricted",
// New versions that use "Post" instead of "Tweet" // New versions that use "Post" instead of "Tweet" and "X" instead of "Twitter"
"This Post was deleted by the Post author. Learn more": "deleted", "This Post was deleted by the Post author. Learn more": "deleted",
"This Post is from a suspended account. Learn more": "suspended", "This Post is from a suspended account. Learn more": "suspended",
"You’re unable to view this Post because this account owner limits who can view their Posts. Learn more": "hidden", "You’re unable to view this Post because this account owner limits who can view their Posts. Learn more": "hidden",
"This Post is unavailable. Learn more": "unavailable", "This Post is unavailable. Learn more": "unavailable",
"This Post violated the Twitter Rules. Learn more": "violated", "This Post violated the X Rules. Learn more": "violated",
"This Post is from an account that no longer exists. Learn more": "no longer exists", "This Post is from an account that no longer exists. Learn more": "no longer exists",
} }

2
pkg/scraper/api_types_v2.go
View File

@ -933,7 +933,7 @@ func (api *API) GetPaginatedQuery(pq PaginatedQuery, count int) (TweetTrove, err
if errors.Is(err, END_OF_FEED) { if errors.Is(err, END_OF_FEED) {
log.Infof("End of feed!") log.Infof("End of feed!")
} else if errors.Is(err, ErrRateLimited) { } else if errors.Is(err, ErrRateLimited) {
log.Errorf("Rate limited!") log.Warnf("Rate limited!")
} else if err != nil { } else if err != nil {
return TweetTrove{}, err return TweetTrove{}, err
} }