me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve closing message when a scrape ends due to rate-limiting

Browse Source
This commit is contained in:
Alessio 2024-06-11 12:32:58 -07:00
parent c454526e47
commit d2045028a7
4 changed files with 24 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/twitter/helpers.go
View File

@ -36,10 +36,10 @@ func die(text string, display_help bool, exit_code int) {
/**
* Print a happy exit message and exit
*/
func happy_exit(text string, was_rate_limited bool) {
if was_rate_limited {
func happy_exit(text string, exit_err error) {
if errors.Is(exit_err, scraper.ErrRateLimited) {
fmt.Printf(terminal_utils.COLOR_YELLOW + text + terminal_utils.COLOR_RESET + "\n")
fmt.Printf(terminal_utils.COLOR_YELLOW + "Exiting early (rate limited)." + terminal_utils.COLOR_RESET + "\n")
fmt.Printf(terminal_utils.COLOR_YELLOW + "Exiting early (%s)." + terminal_utils.COLOR_RESET + "\n", exit_err.Error())
os.Exit(1)
}
fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n")

36
cmd/twitter/main.go
View File

@ -237,7 +237,7 @@ func login(username string, password string) {
}
profile.SaveSession(api)
happy_exit("Logged in as "+string(api.UserHandle), false)
happy_exit("Logged in as "+string(api.UserHandle), nil)
}
/**
@ -272,7 +272,7 @@ func fetch_user(handle scraper.UserHandle) {
}
download_user_content(handle)
happy_exit("Saved the user", false)
happy_exit("Saved the user", nil)
}
/**
@ -297,7 +297,7 @@ func fetch_tweet_only(tweet_identifier string) {
if err2 != nil {
die(fmt.Sprintf("Error saving tweet: %s", err2.Error()), false, 4)
}
happy_exit("Saved the tweet", errors.Is(err, scraper.ErrRateLimited))
happy_exit("Saved the tweet", err)
}
/**
@ -318,7 +318,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
}
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), errors.Is(err, scraper.ErrRateLimited))
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
}
/**
@ -341,7 +341,7 @@ func fetch_user_feed(handle string, how_many int) {
happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited),
err,
)
}
@ -359,7 +359,7 @@ func get_user_likes(handle string, how_many int) {
happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited),
err,
)
}
@ -376,7 +376,7 @@ func get_followees(handle string, how_many int) {
profile.SaveTweetTrove(trove, true)
profile.SaveAsFolloweesList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), errors.Is(err, scraper.ErrRateLimited))
happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err)
}
func get_followers(handle string, how_many int) {
user, err := profile.GetUserByHandle(scraper.UserHandle(handle))
@ -390,7 +390,7 @@ func get_followers(handle string, how_many int) {
profile.SaveTweetTrove(trove, true)
profile.SaveAsFollowersList(user.ID, trove)
happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), errors.Is(err, scraper.ErrRateLimited))
happy_exit(fmt.Sprintf("Saved %d followers", len(trove.Users)), err)
}
func get_bookmarks(how_many int) {
trove, err := scraper.GetBookmarks(how_many)
@ -402,7 +402,7 @@ func get_bookmarks(how_many int) {
happy_exit(fmt.Sprintf(
"Saved %d tweets, %d retweets, %d users, and %d bookmarks",
len(trove.Tweets), len(trove.Retweets), len(trove.Users), len(trove.Bookmarks)),
errors.Is(err, scraper.ErrRateLimited),
err,
)
}
func fetch_timeline(is_following_only bool) {
@ -414,7 +414,7 @@ func fetch_timeline(is_following_only bool) {
happy_exit(
fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)),
errors.Is(err, scraper.ErrRateLimited),
err,
)
}
@ -452,7 +452,7 @@ func search(query string, how_many int) {
}
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), errors.Is(err, scraper.ErrRateLimited))
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)), err)
}
func follow_user(handle string, is_followed bool) {
@ -463,9 +463,9 @@ func follow_user(handle string, is_followed bool) {
profile.SetUserFollowed(&user, is_followed)
if is_followed {
happy_exit("Followed user: "+handle, false)
happy_exit("Followed user: "+handle, nil)
} else {
happy_exit("Unfollowed user: "+handle, false)
happy_exit("Unfollowed user: "+handle, nil)
}
}
@ -478,7 +478,7 @@ func unlike_tweet(tweet_identifier string) {
if err != nil {
die(err.Error(), false, -10)
}
happy_exit("Unliked the tweet.", false)
happy_exit("Unliked the tweet.", nil)
}
func like_tweet(tweet_identifier string) {
@ -494,7 +494,7 @@ func like_tweet(tweet_identifier string) {
if err != nil {
die(err.Error(), false, -1)
}
happy_exit("Liked the tweet.", false)
happy_exit("Liked the tweet.", nil)
}
func list_followed() {
@ -511,7 +511,7 @@ func start_webserver(addr string, should_auto_open bool) {
func fetch_inbox(how_many int) {
trove, _ := scraper.GetInbox(how_many)
profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), false)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
}
func fetch_dm(id string, how_many int) {
@ -524,7 +524,7 @@ func fetch_dm(id string, how_many int) {
profile.SaveDMTrove(trove, true)
happy_exit(
fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)),
errors.Is(err, scraper.ErrRateLimited),
err,
)
}
@ -536,5 +536,5 @@ func send_dm(room_id string, text string, in_reply_to_id int) {
trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id))
profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), false)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)), nil)
}

4
pkg/scraper/api_types.go
View File

@ -373,12 +373,12 @@ var tombstone_types = map[string]string{
"Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, " +
"you’ll need to log in to Twitter. Learn more": "age-restricted",
// New versions that use "Post" instead of "Tweet"
// New versions that use "Post" instead of "Tweet" and "X" instead of "Twitter"
"This Post was deleted by the Post author. Learn more": "deleted",
"This Post is from a suspended account. Learn more": "suspended",
"You’re unable to view this Post because this account owner limits who can view their Posts. Learn more": "hidden",
"This Post is unavailable. Learn more": "unavailable",
"This Post violated the Twitter Rules. Learn more": "violated",
"This Post violated the X Rules. Learn more": "violated",
"This Post is from an account that no longer exists. Learn more": "no longer exists",
}

2
pkg/scraper/api_types_v2.go
View File

@ -933,7 +933,7 @@ func (api *API) GetPaginatedQuery(pq PaginatedQuery, count int) (TweetTrove, err
if errors.Is(err, END_OF_FEED) {
log.Infof("End of feed!")
} else if errors.Is(err, ErrRateLimited) {
log.Errorf("Rate limited!")
log.Warnf("Rate limited!")
} else if err != nil {
return TweetTrove{}, err
}