me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve error handling some more

Browse Source
This commit is contained in:
Alessio 2022-03-06 20:31:04 -08:00
parent d77c55ec1c
commit f41d072573
16 changed files with 62 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.golangci.yaml
View File

@ -26,7 +26,7 @@ linters:
# - wrapcheck # - wrapcheck
- lll - lll
- godox - godox
# - errorlint - errorlint
# # all available settings of specific linters # # all available settings of specific linters
@ -63,13 +63,10 @@ linters-settings:
# - io.Copy(*bytes.Buffer) # - io.Copy(*bytes.Buffer)
# - io.Copy(os.Stdout) # - io.Copy(os.Stdout)
# errorlint: errorlint:
# # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats errorf: true # Ensure Errorf only uses %w (not %v or %s etc) for errors
# errorf: true asserts: true # Require errors.As instead of type-asserting
# # Check for plain type assertions and type switches comparison: true # Require errors.Is instead of equality-checking
# asserts: true
# # Check for plain error comparisons
# comparison: true
# exhaustive: # exhaustive:
# # check switch statements in generated files also # # check switch statements in generated files also

2
cmd/twitter/main.go
View File

@ -230,7 +230,7 @@ func download_tweet_content(tweet_identifier string) {
tweet, err := profile.GetTweetById(tweet_id) tweet, err := profile.GetTweetById(tweet_id)
if err != nil { if err != nil {
panic(fmt.Sprintf("Couldn't get tweet (ID %d) from database: %s", tweet_id, err.Error())) panic(fmt.Errorf("Couldn't get tweet (ID %d) from database:\n %w", tweet_id, err))
} }
err = profile.DownloadTweetContentFor(&tweet) err = profile.DownloadTweetContentFor(&tweet)
if err != nil { if err != nil {

4
persistence/media_download.go
View File

@ -36,12 +36,12 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return fmt.Errorf("Error downloading image %s: %s", url, err.Error()) return fmt.Errorf("Error downloading image %s:\n %w", url, err)
} }
err = os.WriteFile(outpath, data, 0644) err = os.WriteFile(outpath, data, 0644)
if err != nil { if err != nil {
return fmt.Errorf("Error writing to path: %s, url: %s: %s", outpath, url, err.Error()) return fmt.Errorf("Error writing to path %s, url %s:\n %w", outpath, url, err)
} }
return nil return nil
} }

13
persistence/profile.go
View File

@ -22,16 +22,7 @@ type Profile struct {
DB *sql.DB DB *sql.DB
} }
/** var ErrTargetAlreadyExists = fmt.Errorf("Target already exists")
* Custom error
*/
type ErrTargetAlreadyExists struct {
target string
}
func (err ErrTargetAlreadyExists) Error() string {
return fmt.Sprintf("Target already exists: %s", err.target)
}
/** /**
* Create a new profile in the given location. * Create a new profile in the given location.
@ -45,7 +36,7 @@ func (err ErrTargetAlreadyExists) Error() string {
*/ */
func NewProfile(target_dir string) (Profile, error) { func NewProfile(target_dir string) (Profile, error) {
if file_exists(target_dir) { if file_exists(target_dir) {
return Profile{}, ErrTargetAlreadyExists{target_dir} return Profile{}, fmt.Errorf("Could not create target %q:\n %w", target_dir, ErrTargetAlreadyExists)
} }
settings_file := path.Join(target_dir, "settings.yaml") settings_file := path.Join(target_dir, "settings.yaml")

3
persistence/profile_test.go
View File

@ -40,8 +40,7 @@ func TestNewProfileInvalidPath(t *testing.T) {
_, err = persistence.NewProfile(gibberish_path) _, err = persistence.NewProfile(gibberish_path)
require.Error(err, "Should have failed to create a profile in an already existing directory!") require.Error(err, "Should have failed to create a profile in an already existing directory!")
_, is_right_type := err.(persistence.ErrTargetAlreadyExists) assert.ErrorIs(t, err, persistence.ErrTargetAlreadyExists)
assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err)
} }
/** /**

5
persistence/tweet_queries.go
View File

@ -3,6 +3,7 @@ package persistence
import ( import (
"database/sql" "database/sql"
"strings" "strings"
"errors"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -82,7 +83,7 @@ func (p Profile) IsTweetInDatabase(id scraper.TweetID) bool {
var dummy string var dummy string
err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy) err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy)
if err != nil { if err != nil {
if err != sql.ErrNoRows { if !errors.Is(err, sql.ErrNoRows) {
// A real error // A real error
panic(err) panic(err)
} }
@ -189,7 +190,7 @@ func (p Profile) CheckTweetContentDownloadNeeded(tweet scraper.Tweet) bool {
var is_content_downloaded bool var is_content_downloaded bool
err := row.Scan(&is_content_downloaded) err := row.Scan(&is_content_downloaded)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return true return true
} else { } else {
panic(err) panic(err)

10
persistence/tweet_trove_queries.go
View File

@ -15,12 +15,12 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
// Download download their tiny profile image // Download download their tiny profile image
err := p.DownloadUserProfileImageTiny(&u) err := p.DownloadUserProfileImageTiny(&u)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error downloading user content for user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error())) panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
} }
err = p.SaveUser(&u) err = p.SaveUser(&u)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error saving user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error())) panic(fmt.Errorf("Error saving user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
} }
fmt.Println(u.Handle, u.ID) fmt.Println(u.Handle, u.ID)
// If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too // If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too
@ -33,19 +33,19 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
for _, t := range trove.Tweets { for _, t := range trove.Tweets {
err := p.SaveTweet(t) err := p.SaveTweet(t)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error saving tweet ID %d: %s", t.ID, err.Error())) panic(fmt.Errorf("Error saving tweet ID %d:\n %w", t.ID, err))
} }
err = p.DownloadTweetContentFor(&t) err = p.DownloadTweetContentFor(&t)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error downloading tweet content for tweet ID %d: %s", t.ID, err.Error())) panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n %w", t.ID, err))
} }
} }
for _, r := range trove.Retweets { for _, r := range trove.Retweets {
err := p.SaveRetweet(r) err := p.SaveRetweet(r)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error saving retweet with ID %d from user ID %d: %s", r.RetweetID, r.RetweetedByID, err.Error())) panic(fmt.Errorf("Error saving retweet with ID %d from user ID %d:\n %w", r.RetweetID, r.RetweetedByID, err))
} }
} }
} }

19
persistence/user_queries.go
View File

@ -2,6 +2,7 @@ package persistence
import ( import (
"fmt" "fmt"
"errors"
"database/sql" "database/sql"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -16,7 +17,7 @@ import (
func (p Profile) SaveUser(u *scraper.User) error { func (p Profile) SaveUser(u *scraper.User) error {
if u.IsNeedingFakeID { if u.IsNeedingFakeID {
err := p.DB.QueryRow("select id from users where lower(handle) = lower(?)", u.Handle).Scan(&u.ID) err := p.DB.QueryRow("select id from users where lower(handle) = lower(?)", u.Handle).Scan(&u.ID)
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
// We need to continue-- create a new fake user // We need to continue-- create a new fake user
u.ID = p.NextFakeUserID() u.ID = p.NextFakeUserID()
} else if err == nil { } else if err == nil {
@ -24,7 +25,7 @@ func (p Profile) SaveUser(u *scraper.User) error {
return nil return nil
} else { } else {
// A real error occurred // A real error occurred
panic(fmt.Sprintf("Error checking for existence of fake user with handle %q: %s", u.Handle, err.Error())) panic(fmt.Errorf("Error checking for existence of fake user with handle %q:\n %w", u.Handle, err))
} }
} }
@ -79,7 +80,7 @@ func (p Profile) UserExists(handle scraper.UserHandle) bool {
var dummy string var dummy string
err := db.QueryRow("select 1 from users where lower(handle) = lower(?)", handle).Scan(&dummy) err := db.QueryRow("select 1 from users where lower(handle) = lower(?)", handle).Scan(&dummy)
if err != nil { if err != nil {
if err != sql.ErrNoRows { if !errors.Is(err, sql.ErrNoRows) {
// A real error // A real error
panic(err) panic(err)
} }
@ -109,7 +110,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
where lower(handle) = lower(?) where lower(handle) = lower(?)
`, handle) `, handle)
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return ret, ErrNotInDatabase{"User", handle} return ret, ErrNotInDatabase{"User", handle}
} }
return ret, nil return ret, nil
@ -136,7 +137,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
from users from users
where id = ? where id = ?
`, id) `, id)
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return ret, ErrNotInDatabase{"User", id} return ret, ErrNotInDatabase{"User", id}
} }
return ret, err return ret, err
@ -164,7 +165,7 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
var banner_image_url string var banner_image_url string
err := row.Scan(&is_content_downloaded, &profile_image_url, &banner_image_url) err := row.Scan(&is_content_downloaded, &profile_image_url, &banner_image_url)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return true return true
} else { } else {
panic(err) panic(err)
@ -189,14 +190,14 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
func (p Profile) SetUserFollowed(user *scraper.User, is_followed bool) { func (p Profile) SetUserFollowed(user *scraper.User, is_followed bool) {
result, err := p.DB.Exec("update users set is_followed = ? where id = ?", is_followed, user.ID) result, err := p.DB.Exec("update users set is_followed = ? where id = ?", is_followed, user.ID)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error inserting user with handle %q: %s", user.Handle, err.Error())) panic(fmt.Errorf("Error inserting user with handle %q:\n %w", user.Handle, err))
} }
count, err := result.RowsAffected() count, err := result.RowsAffected()
if err != nil { if err != nil {
panic("Unknown error: " + err.Error()) panic(fmt.Errorf("Unknown error retrieving row count:\n %w", err))
} }
if count != 1 { if count != 1 {
panic(fmt.Sprintf("User with handle %q not found", user.Handle)) panic(fmt.Errorf("User with handle %q not found", user.Handle))
} }
user.IsFollowed = is_followed user.IsFollowed = is_followed
} }

6
persistence/user_queries_test.go
View File

@ -233,16 +233,16 @@ func TestCreateUnknownUserWithHandle(t *testing.T) {
err := profile.SaveUser(&user) err := profile.SaveUser(&user)
assert.NoError(err) assert.NoError(err)
assert.Equal(scraper.UserID(next_id + 1), user.ID) assert.Equal(scraper.UserID(next_id+1), user.ID)
// Ensure the change was persisted // Ensure the change was persisted
user_reloaded, err := profile.GetUserByHandle(user.Handle) user_reloaded, err := profile.GetUserByHandle(user.Handle)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(handle, user_reloaded.Handle) // Verify it's the same user assert.Equal(handle, user_reloaded.Handle) // Verify it's the same user
assert.Equal(scraper.UserID(next_id + 1), user_reloaded.ID) assert.Equal(scraper.UserID(next_id+1), user_reloaded.ID)
// Why not tack this test on here: make sure NextFakeUserID works as expected // Why not tack this test on here: make sure NextFakeUserID works as expected
assert.Equal(next_id + 2, profile.NextFakeUserID()) assert.Equal(next_id+2, profile.NextFakeUserID())
} }
/** /**

8
scraper/api_errors.go
View File

@ -4,5 +4,9 @@ import (
"fmt" "fmt"
) )
var END_OF_FEED = fmt.Errorf("End of feed") var (
var DOESNT_EXIST = fmt.Errorf("Doesn't exist") END_OF_FEED = fmt.Errorf("End of feed")
DOESNT_EXIST = fmt.Errorf("Doesn't exist")
EXTERNAL_API_ERROR = fmt.Errorf("Unexpected result from external API")
API_PARSE_ERROR = fmt.Errorf("Couldn't parse the result returned from the API")
)

4
scraper/api_types.go
View File

@ -271,7 +271,7 @@ func (u UserResponse) ConvertToAPIUser() APIUser {
} else if api_error.Name == "NotFoundError" { } else if api_error.Name == "NotFoundError" {
ret.DoesntExist = true ret.DoesntExist = true
} else { } else {
panic(fmt.Sprintf("Unknown api error: %q", api_error.Message)) panic(fmt.Errorf("Unknown api error %q:\n %w", api_error.Message, EXTERNAL_API_ERROR))
} }
} }
@ -401,7 +401,7 @@ func (t *TweetResponse) HandleTombstones() []UserHandle {
short_text, ok := tombstone_types[entry.GetTombstoneText()] short_text, ok := tombstone_types[entry.GetTombstoneText()]
if !ok { if !ok {
panic(fmt.Sprintf("Unknown tombstone text: %s", entry.GetTombstoneText())) panic(fmt.Errorf("Unknown tombstone text %q:\n %w", entry.GetTombstoneText(), EXTERNAL_API_ERROR))
} }
tombstoned_tweet.TombstoneText = short_text tombstoned_tweet.TombstoneText = short_text

6
scraper/api_types_v2.go
View File

@ -186,7 +186,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
var ok bool var ok bool
tombstoned_tweet.TombstoneText, ok = tombstone_types[quoted_api_result.Result.Tombstone.Text.Text] tombstoned_tweet.TombstoneText, ok = tombstone_types[quoted_api_result.Result.Tombstone.Text.Text]
if !ok { if !ok {
panic(fmt.Sprintf("Unknown tombstone text: %s", quoted_api_result.Result.Tombstone.Text.Text)) panic(fmt.Errorf("Unknown tombstone text %q:\n %w", quoted_api_result.Result.Tombstone.Text.Text, EXTERNAL_API_ERROR))
} }
tombstoned_tweet.ID = int64(int_or_panic(api_result.Result.Legacy.APITweet.QuotedStatusIDStr)) tombstoned_tweet.ID = int64(int_or_panic(api_result.Result.Legacy.APITweet.QuotedStatusIDStr))
handle, err := ParseHandleFromTweetUrl(api_result.Result.Legacy.APITweet.QuotedStatusPermalink.ExpandedURL) handle, err := ParseHandleFromTweetUrl(api_result.Result.Legacy.APITweet.QuotedStatusPermalink.ExpandedURL)
@ -209,7 +209,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
// and the retweeted TweetResults; it should only be parsed for the real Tweet, not the Retweet // and the retweeted TweetResults; it should only be parsed for the real Tweet, not the Retweet
main_tweet, ok := ret.Tweets[TweetID(api_result.Result.Legacy.ID)] main_tweet, ok := ret.Tweets[TweetID(api_result.Result.Legacy.ID)]
if !ok { if !ok {
panic(fmt.Sprintf("Tweet trove didn't contain its own tweet: %d", api_result.Result.Legacy.ID)) panic(fmt.Errorf("Tweet trove didn't contain its own tweet with ID %d:\n %w", api_result.Result.Legacy.ID, EXTERNAL_API_ERROR))
} }
if api_result.Result.Card.Legacy.Name == "summary_large_image" || api_result.Result.Card.Legacy.Name == "player" { if api_result.Result.Card.Legacy.Name == "summary_large_image" || api_result.Result.Card.Legacy.Name == "player" {
url := api_result.Result.Card.ParseAsUrl() url := api_result.Result.Card.ParseAsUrl()
@ -225,7 +225,7 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove
main_tweet.Urls[i] = url main_tweet.Urls[i] = url
} }
if !found { if !found {
panic(fmt.Sprintf("Couldn't find the url in tweet ID: %d", api_result.Result.Legacy.ID)) panic(fmt.Errorf("Couldn't find the url in tweet ID %d:\n %w", api_result.Result.Legacy.ID, EXTERNAL_API_ERROR))
} }
} else if strings.Index(api_result.Result.Card.Legacy.Name, "poll") == 0 { } else if strings.Index(api_result.Result.Card.Legacy.Name, "poll") == 0 {
// Process polls // Process polls

4
scraper/link_expander.go
View File

@ -24,12 +24,12 @@ func ExpandShortUrl(short_url string) string {
panic(err) // TODO: handle timeouts panic(err) // TODO: handle timeouts
} }
if resp.StatusCode != 301 { if resp.StatusCode != 301 {
panic(fmt.Sprintf("Unknown status code returned when expanding short url %q: %s", short_url, resp.Status)) panic(fmt.Errorf("Unknown status code returned when expanding short url %q: %s\n %w", short_url, resp.Status, EXTERNAL_API_ERROR))
} }
long_url := resp.Header.Get("Location") long_url := resp.Header.Get("Location")
if long_url == "" { if long_url == "" {
panic(fmt.Sprintf("Header didn't have a Location field for short url %q", short_url)) panic(fmt.Errorf("Header didn't have a Location field for short url %q:\n %w", short_url, EXTERNAL_API_ERROR))
} }
return long_url return long_url
} }

8
scraper/tweet.go
View File

@ -126,7 +126,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
// Process images // Process images
for _, media := range apiTweet.Entities.Media { for _, media := range apiTweet.Entities.Media {
if media.Type != "photo" { // TODO: remove this eventually if media.Type != "photo" { // TODO: remove this eventually
panic(fmt.Sprintf("Unknown media type: %q", media.Type)) panic(fmt.Errorf("Unknown media type %q:\n %w", media.Type, EXTERNAL_API_ERROR))
} }
new_image := ParseAPIMedia(media) new_image := ParseAPIMedia(media)
new_image.TweetID = ret.ID new_image.TweetID = ret.ID
@ -145,7 +145,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
for _, mention := range strings.Split(apiTweet.Entities.ReplyMentions, " ") { for _, mention := range strings.Split(apiTweet.Entities.ReplyMentions, " ") {
if mention != "" { if mention != "" {
if mention[0] != '@' { if mention[0] != '@' {
panic(fmt.Sprintf("Unknown ReplyMention value: %s", apiTweet.Entities.ReplyMentions)) panic(fmt.Errorf("Unknown ReplyMention value %q:\n %w", apiTweet.Entities.ReplyMentions, EXTERNAL_API_ERROR))
} }
ret.ReplyMentions = append(ret.ReplyMentions, UserHandle(mention[1:])) ret.ReplyMentions = append(ret.ReplyMentions, UserHandle(mention[1:]))
} }
@ -158,7 +158,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
continue continue
} }
if len(apiTweet.ExtendedEntities.Media) != 1 { if len(apiTweet.ExtendedEntities.Media) != 1 {
panic(fmt.Sprintf("Surprising ExtendedEntities: %v", apiTweet.ExtendedEntities.Media)) panic(fmt.Errorf("Surprising ExtendedEntities: %v\n %w", apiTweet.ExtendedEntities.Media, EXTERNAL_API_ERROR))
} }
new_video := ParseAPIVideo(apiTweet.ExtendedEntities.Media[0], ret.ID) new_video := ParseAPIVideo(apiTweet.ExtendedEntities.Media[0], ret.ID)
ret.Videos = []Video{new_video} ret.Videos = []Video{new_video}
@ -194,7 +194,7 @@ func GetTweet(id TweetID) (Tweet, error) {
api := API{} api := API{}
tweet_response, err := api.GetTweet(id, "") tweet_response, err := api.GetTweet(id, "")
if err != nil { if err != nil {
return Tweet{}, fmt.Errorf("Error in API call: %s", err) return Tweet{}, fmt.Errorf("Error in API call:\n %w", err)
} }
single_tweet, ok := tweet_response.GlobalObjects.Tweets[fmt.Sprint(id)] single_tweet, ok := tweet_response.GlobalObjects.Tweets[fmt.Sprint(id)]

8
scraper/tweet_trove.go
View File

@ -83,7 +83,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() {
log.Debug("Getting tombstone user: " + handle) log.Debug("Getting tombstone user: " + handle)
user, err := GetUser(handle) user, err := GetUser(handle)
if err != nil { if err != nil {
panic(fmt.Sprintf("Error getting tombstoned user: %s\n %s", handle, err.Error())) panic(fmt.Errorf("Error getting tombstoned user with handle %q: \n %w", handle, err))
} }
if user.ID == 0 { if user.ID == 0 {
@ -124,7 +124,11 @@ func (trove *TweetTrove) FillMissingUserIDs() {
if !is_found { if !is_found {
// The user probably deleted deleted their account, and thus `scraper.GetUser` failed. So // The user probably deleted deleted their account, and thus `scraper.GetUser` failed. So
// they're not in this trove's Users. // they're not in this trove's Users.
panic(fmt.Sprintf("Couldn't fill out this Tweet's UserID: %d, %s", tweet.ID, tweet.UserHandle)) panic(fmt.Errorf(
"Couldn't find user ID for user %q, while filling missing UserID in tweet with ID %d",
tweet.UserHandle,
tweet.ID,
))
} }
tweet.UserID = user.ID tweet.UserID = user.ID
trove.Tweets[i] = tweet trove.Tweets[i] = tweet

2
scraper/user.go
View File

@ -207,7 +207,7 @@ func (u User) GetTinyProfileImageUrl() string {
// Check that the format is as expected // Check that the format is as expected
r := regexp.MustCompile(`(\.\w{2,4})$`) r := regexp.MustCompile(`(\.\w{2,4})$`)
if !r.MatchString(u.ProfileImageUrl) { if !r.MatchString(u.ProfileImageUrl) {
panic(fmt.Sprintf("Weird profile image url: %s", u.ProfileImageUrl)) panic(fmt.Errorf("Weird profile image url (here is the file extension?): %s", u.ProfileImageUrl))
} }
return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1") return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1")
} }