me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve error handling some more

Browse Source
This commit is contained in:
Alessio 2022-03-06 20:31:04 -08:00
parent d77c55ec1c
commit f41d072573
16 changed files with 62 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.golangci.yaml
View File

@ -26,7 +26,7 @@ linters:
# - wrapcheck
- lll
- godox
# - errorlint
- errorlint
# # all available settings of specific linters
@ -63,13 +63,10 @@ linters-settings:
# - io.Copy(*bytes.Buffer)
# - io.Copy(os.Stdout)
# errorlint:
# # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats
# errorf: true
# # Check for plain type assertions and type switches
# asserts: true
# # Check for plain error comparisons
# comparison: true
errorlint:
errorf: true # Ensure Errorf only uses %w (not %v or %s etc) for errors
asserts: true # Require errors.As instead of type-asserting
comparison: true # Require errors.Is instead of equality-checking
# exhaustive:
# # check switch statements in generated files also

2
cmd/twitter/main.go
View File

@ -230,7 +230,7 @@ func download_tweet_content(tweet_identifier string) {
tweet, err := profile.GetTweetById(tweet_id)
if err != nil {
panic(fmt.Sprintf("Couldn't get tweet (ID %d) from database: %s", tweet_id, err.Error()))
panic(fmt.Errorf("Couldn't get tweet (ID %d) from database:\n %w", tweet_id, err))
}
err = profile.DownloadTweetContentFor(&tweet)
if err != nil {

4
persistence/media_download.go
View File

@ -36,12 +36,12 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("Error downloading image %s: %s", url, err.Error())
return fmt.Errorf("Error downloading image %s:\n %w", url, err)
}
err = os.WriteFile(outpath, data, 0644)
if err != nil {
return fmt.Errorf("Error writing to path: %s, url: %s: %s", outpath, url, err.Error())
return fmt.Errorf("Error writing to path %s, url %s:\n %w", outpath, url, err)
}
return nil
}

13
persistence/profile.go
View File

@ -22,16 +22,7 @@ type Profile struct {
DB *sql.DB
}
/**
* Custom error
*/
type ErrTargetAlreadyExists struct {
target string
}
func (err ErrTargetAlreadyExists) Error() string {
return fmt.Sprintf("Target already exists: %s", err.target)
}
var ErrTargetAlreadyExists = fmt.Errorf("Target already exists")
/**
* Create a new profile in the given location.
@ -45,7 +36,7 @@ func (err ErrTargetAlreadyExists) Error() string {
*/
func NewProfile(target_dir string) (Profile, error) {
if file_exists(target_dir) {
return Profile{}, ErrTargetAlreadyExists{target_dir}
return Profile{}, fmt.Errorf("Could not create target %q:\n %w", target_dir, ErrTargetAlreadyExists)
}
settings_file := path.Join(target_dir, "settings.yaml")

3
persistence/profile_test.go
View File

@ -40,8 +40,7 @@ func TestNewProfileInvalidPath(t *testing.T) {
_, err = persistence.NewProfile(gibberish_path)
require.Error(err, "Should have failed to create a profile in an already existing directory!")
_, is_right_type := err.(persistence.ErrTargetAlreadyExists)
assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err)
assert.ErrorIs(t, err, persistence.ErrTargetAlreadyExists)
}
/**

5
persistence/tweet_queries.go
View File

@ -3,6 +3,7 @@ package persistence
import (
"database/sql"
"strings"
"errors"
"offline_twitter/scraper"
)
@ -82,7 +83,7 @@ func (p Profile) IsTweetInDatabase(id scraper.TweetID) bool {
var dummy string
err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy)
if err != nil {
if err != sql.ErrNoRows {
if !errors.Is(err, sql.ErrNoRows) {
// A real error
panic(err)
}
@ -189,7 +190,7 @@ func (p Profile) CheckTweetContentDownloadNeeded(tweet scraper.Tweet) bool {
var is_content_downloaded bool
err := row.Scan(&is_content_downloaded)
if err != nil {
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return true
} else {
panic(err)

10
persistence/tweet_trove_queries.go
View File

@ -15,12 +15,12 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
// Download download their tiny profile image
err := p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Sprintf("Error downloading user content for user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error()))
panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
}
err = p.SaveUser(&u)
if err != nil {
panic(fmt.Sprintf("Error saving user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error()))
panic(fmt.Errorf("Error saving user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
}
fmt.Println(u.Handle, u.ID)
// If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too
@ -33,19 +33,19 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
for _, t := range trove.Tweets {
err := p.SaveTweet(t)
if err != nil {
panic(fmt.Sprintf("Error saving tweet ID %d: %s", t.ID, err.Error()))
panic(fmt.Errorf("Error saving tweet ID %d:\n %w", t.ID, err))
}
err = p.DownloadTweetContentFor(&t)
if err != nil {
panic(fmt.Sprintf("Error downloading tweet content for tweet ID %d: %s", t.ID, err.Error()))
panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n %w", t.ID, err))
}
}
for _, r := range trove.Retweets {
err := p.SaveRetweet(r)
if err != nil {
panic(fmt.Sprintf("Error saving retweet with ID %d from user ID %d: %s", r.RetweetID, r.RetweetedByID, err.Error()))
panic(fmt.Errorf("Error saving retweet with ID %d from user ID %d:\n %w", r.RetweetID, r.RetweetedByID, err))
}
}
}

19
persistence/user_queries.go
View File

@ -2,6 +2,7 @@ package persistence
import (
"fmt"
"errors"
"database/sql"
"offline_twitter/scraper"
)
@ -16,7 +17,7 @@ import (
func (p Profile) SaveUser(u *scraper.User) error {
if u.IsNeedingFakeID {
err := p.DB.QueryRow("select id from users where lower(handle) = lower(?)", u.Handle).Scan(&u.ID)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We need to continue-- create a new fake user
u.ID = p.NextFakeUserID()
} else if err == nil {
@ -24,7 +25,7 @@ func (p Profile) SaveUser(u *scraper.User) error {
return nil
} else {
// A real error occurred
panic(fmt.Sprintf("Error checking for existence of fake user with handle %q: %s", u.Handle, err.Error()))
panic(fmt.Errorf("Error checking for existence of fake user with handle %q:\n %w", u.Handle, err))
}
}
@ -79,7 +80,7 @@ func (p Profile) UserExists(handle scraper.UserHandle) bool {
var dummy string
err := db.QueryRow("select 1 from users where lower(handle) = lower(?)", handle).Scan(&dummy)
if err != nil {
if err != sql.ErrNoRows {
if !errors.Is(err, sql.ErrNoRows) {
// A real error
panic(err)
}
@ -109,7 +110,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
where lower(handle) = lower(?)
`, handle)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return ret, ErrNotInDatabase{"User", handle}
}
return ret, nil
@ -136,7 +137,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
from users
where id = ?
`, id)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return ret, ErrNotInDatabase{"User", id}
}
return ret, err
@ -164,7 +165,7 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
var banner_image_url string
err := row.Scan(&is_content_downloaded, &profile_image_url, &banner_image_url)
if err != nil {
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return true
} else {
panic(err)
@ -189,14 +190,14 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
func (p Profile) SetUserFollowed(user *scraper.User, is_followed bool) {
result, err := p.DB.Exec("update users set is_followed = ? where id = ?", is_followed, user.ID)
if err != nil {
panic(fmt.Sprintf("Error inserting user with handle %q: %s", user.Handle, err.Error()))
panic(fmt.Errorf("Error inserting user with handle %q:\n %w", user.Handle, err))
}
count, err := result.RowsAffected()
if err != nil {
panic("Unknown error: " + err.Error())
panic(fmt.Errorf("Unknown error retrieving row count:\n %w", err))
}
if count != 1 {
panic(fmt.Sprintf("User with handle %q not found", user.Handle))
panic(fmt.Errorf("User with handle %q not found", user.Handle))
}
user.IsFollowed = is_followed
}

8
scraper/api_errors.go
View File

@ -4,5 +4,9 @@ import (
"fmt"
)
var END_OF_FEED = fmt.Errorf("End of feed")
var DOESNT_EXIST = fmt.Errorf("Doesn't exist")
var (
END_OF_FEED = fmt.Errorf("End of feed")
DOESNT_EXIST = fmt.Errorf("Doesn't exist")
EXTERNAL_API_ERROR = fmt.Errorf("Unexpected result from external API")
API_PARSE_ERROR = fmt.Errorf("Couldn't parse the result returned from the API")
)

4
scraper/api_types.go
View File

@ -271,7 +271,7 @@ func (u UserResponse) ConvertToAPIUser() APIUser {
} else if api_error.Name == "NotFoundError" {
ret.DoesntExist = true
} else {
panic(fmt.Sprintf("Unknown api error: %q", api_error.Message))
panic(fmt.Errorf("Unknown api error %q:\n %w", api_error.Message, EXTERNAL_API_ERROR))
}
}
@ -401,7 +401,7 @@ func (t *TweetResponse) HandleTombstones() []UserHandle {
short_text, ok := tombstone_types[entry.GetTombstoneText()]
if !ok {
panic(fmt.Sprintf("Unknown tombstone text: %s", entry.GetTombstoneText()))
panic(fmt.Errorf("Unknown tombstone text %q:\n %w", entry.GetTombstoneText(), EXTERNAL_API_ERROR))
}
tombstoned_tweet.TombstoneText = short_text

6
scraper/api_types_v2.go
View File

@ -186,7 +186,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
var ok bool
tombstoned_tweet.TombstoneText, ok = tombstone_types[quoted_api_result.Result.Tombstone.Text.Text]
if !ok {
panic(fmt.Sprintf("Unknown tombstone text: %s", quoted_api_result.Result.Tombstone.Text.Text))
panic(fmt.Errorf("Unknown tombstone text %q:\n %w", quoted_api_result.Result.Tombstone.Text.Text, EXTERNAL_API_ERROR))
}
tombstoned_tweet.ID = int64(int_or_panic(api_result.Result.Legacy.APITweet.QuotedStatusIDStr))
handle, err := ParseHandleFromTweetUrl(api_result.Result.Legacy.APITweet.QuotedStatusPermalink.ExpandedURL)
@ -209,7 +209,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
// and the retweeted TweetResults; it should only be parsed for the real Tweet, not the Retweet
main_tweet, ok := ret.Tweets[TweetID(api_result.Result.Legacy.ID)]
if !ok {
panic(fmt.Sprintf("Tweet trove didn't contain its own tweet: %d", api_result.Result.Legacy.ID))
panic(fmt.Errorf("Tweet trove didn't contain its own tweet with ID %d:\n %w", api_result.Result.Legacy.ID, EXTERNAL_API_ERROR))
}
if api_result.Result.Card.Legacy.Name == "summary_large_image" || api_result.Result.Card.Legacy.Name == "player" {
url := api_result.Result.Card.ParseAsUrl()
@ -225,7 +225,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
main_tweet.Urls[i] = url
}
if !found {
panic(fmt.Sprintf("Couldn't find the url in tweet ID: %d", api_result.Result.Legacy.ID))
panic(fmt.Errorf("Couldn't find the url in tweet ID %d:\n %w", api_result.Result.Legacy.ID, EXTERNAL_API_ERROR))
}
} else if strings.Index(api_result.Result.Card.Legacy.Name, "poll") == 0 {
// Process polls

4
scraper/link_expander.go
View File

@ -24,12 +24,12 @@ func ExpandShortUrl(short_url string) string {
panic(err) // TODO: handle timeouts
}
if resp.StatusCode != 301 {
panic(fmt.Sprintf("Unknown status code returned when expanding short url %q: %s", short_url, resp.Status))
panic(fmt.Errorf("Unknown status code returned when expanding short url %q: %s\n %w", short_url, resp.Status, EXTERNAL_API_ERROR))
}
long_url := resp.Header.Get("Location")
if long_url == "" {
panic(fmt.Sprintf("Header didn't have a Location field for short url %q", short_url))
panic(fmt.Errorf("Header didn't have a Location field for short url %q:\n %w", short_url, EXTERNAL_API_ERROR))
}
return long_url
}

8
scraper/tweet.go
View File

@ -126,7 +126,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
// Process images
for _, media := range apiTweet.Entities.Media {
if media.Type != "photo" { // TODO: remove this eventually
panic(fmt.Sprintf("Unknown media type: %q", media.Type))
panic(fmt.Errorf("Unknown media type %q:\n %w", media.Type, EXTERNAL_API_ERROR))
}
new_image := ParseAPIMedia(media)
new_image.TweetID = ret.ID
@ -145,7 +145,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
for _, mention := range strings.Split(apiTweet.Entities.ReplyMentions, " ") {
if mention != "" {
if mention[0] != '@' {
panic(fmt.Sprintf("Unknown ReplyMention value: %s", apiTweet.Entities.ReplyMentions))
panic(fmt.Errorf("Unknown ReplyMention value %q:\n %w", apiTweet.Entities.ReplyMentions, EXTERNAL_API_ERROR))
}
ret.ReplyMentions = append(ret.ReplyMentions, UserHandle(mention[1:]))
}
@ -158,7 +158,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
continue
}
if len(apiTweet.ExtendedEntities.Media) != 1 {
panic(fmt.Sprintf("Surprising ExtendedEntities: %v", apiTweet.ExtendedEntities.Media))
panic(fmt.Errorf("Surprising ExtendedEntities: %v\n %w", apiTweet.ExtendedEntities.Media, EXTERNAL_API_ERROR))
}
new_video := ParseAPIVideo(apiTweet.ExtendedEntities.Media[0], ret.ID)
ret.Videos = []Video{new_video}
@ -194,7 +194,7 @@ func GetTweet(id TweetID) (Tweet, error) {
api := API{}
tweet_response, err := api.GetTweet(id, "")
if err != nil {
return Tweet{}, fmt.Errorf("Error in API call: %s", err)
return Tweet{}, fmt.Errorf("Error in API call:\n %w", err)
}
single_tweet, ok := tweet_response.GlobalObjects.Tweets[fmt.Sprint(id)]

8
scraper/tweet_trove.go
View File

@ -83,7 +83,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() {
log.Debug("Getting tombstone user: " + handle)
user, err := GetUser(handle)
if err != nil {
panic(fmt.Sprintf("Error getting tombstoned user: %s\n %s", handle, err.Error()))
panic(fmt.Errorf("Error getting tombstoned user with handle %q: \n %w", handle, err))
}
if user.ID == 0 {
@ -124,7 +124,11 @@ func (trove *TweetTrove) FillMissingUserIDs() {
if !is_found {
// The user probably deleted deleted their account, and thus `scraper.GetUser` failed. So
// they're not in this trove's Users.
panic(fmt.Sprintf("Couldn't fill out this Tweet's UserID: %d, %s", tweet.ID, tweet.UserHandle))
panic(fmt.Errorf(
"Couldn't find user ID for user %q, while filling missing UserID in tweet with ID %d",
tweet.UserHandle,
tweet.ID,
))
}
tweet.UserID = user.ID
trove.Tweets[i] = tweet

2
scraper/user.go
View File

@ -207,7 +207,7 @@ func (u User) GetTinyProfileImageUrl() string {
// Check that the format is as expected
r := regexp.MustCompile(`(\.\w{2,4})$`)
if !r.MatchString(u.ProfileImageUrl) {
panic(fmt.Sprintf("Weird profile image url: %s", u.ProfileImageUrl))
panic(fmt.Errorf("Weird profile image url (here is the file extension?): %s", u.ProfileImageUrl))
}
return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1")
}