diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index ec963e8..87605a3 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -288,7 +288,15 @@ func create_profile(target_dir string) { */ func fetch_user(handle scraper.UserHandle) { user, err := scraper.GetUser(handle) - if is_scrape_failure(err) { + if errors.Is(err, scraper.ErrDoesntExist) { + // There's several reasons we could get a ErrDoesntExist: + // 1. account never existed (user made a CLI typo) + // 2. user changed their handle + // 3. user deleted their account + // In case (1), we should just report the error; in case (2) and (3), it would be nice to rescrape by ID, + // but that feels kind of too complicated to do here. So just report the error and let the user decide + die(fmt.Sprintf("User with handle %q doesn't exist. Check spelling, or try scraping with the ID instead", handle), false, -1) + } else if is_scrape_failure(err) { die(err.Error(), false, -1) } log.Debug(user) diff --git a/internal/webserver/handler_login.go b/internal/webserver/handler_login.go index c945770..eb6755a 100644 --- a/internal/webserver/handler_login.go +++ b/internal/webserver/handler_login.go @@ -70,7 +70,7 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api // Ensure the user is downloaded user, err := scraper.GetUser(api.UserHandle) - if err != nil { + if err != nil { // ErrDoesntExist or otherwise app.error_404(w, r) return } diff --git a/internal/webserver/handler_user_feed.go b/internal/webserver/handler_user_feed.go index ad9569d..1d75b51 100644 --- a/internal/webserver/handler_user_feed.go +++ b/internal/webserver/handler_user_feed.go @@ -16,16 +16,18 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[0])) - if err != nil { + if errors.Is(err, persistence.ErrNotInDatabase) { if !app.IsScrapingDisabled { user, err = scraper.GetUser(scraper.UserHandle(parts[0])) } - if err != nil { + if err != nil { // ErrDoesntExist or otherwise app.error_404(w, r) return } panic_if(app.Profile.SaveUser(&user)) panic_if(app.Profile.DownloadUserContentFor(&user, &app.API)) + } else if err != nil { + panic(err) } if len(parts) > 1 && parts[1] == "followers" { diff --git a/pkg/scraper/api_types.go b/pkg/scraper/api_types.go index 28a5bb5..7c28439 100644 --- a/pkg/scraper/api_types.go +++ b/pkg/scraper/api_types.go @@ -295,6 +295,7 @@ type UserResponse struct { Data struct { User struct { Result struct { + MetaTypename string `json:"__typename"` ID int64 `json:"rest_id,string"` Legacy APIUser `json:"legacy"` IsBlueVerified bool `json:"is_blue_verified"` @@ -312,7 +313,12 @@ type UserResponse struct { } `json:"errors"` } -func (u UserResponse) ConvertToAPIUser() APIUser { +func (u UserResponse) ConvertToAPIUser() (APIUser, error) { + if u.Data.User.Result.MetaTypename == "" { + // Completely empty response (user not found) + return APIUser{}, ErrDoesntExist + } + ret := u.Data.User.Result.Legacy ret.ID = u.Data.User.Result.ID ret.Verified = u.Data.User.Result.IsBlueVerified @@ -338,7 +344,7 @@ func (u UserResponse) ConvertToAPIUser() APIUser { ret.DoesntExist = true } - return ret + return ret, nil } type Entry struct { diff --git a/pkg/scraper/api_types_test.go b/pkg/scraper/api_types_test.go index 731bfad..01dc7a9 100644 --- a/pkg/scraper/api_types_test.go +++ b/pkg/scraper/api_types_test.go @@ -69,7 +69,8 @@ func TestUserProfileToAPIUser(t *testing.T) { err = json.Unmarshal(data, &user_resp) assert.NoError(err) - result := user_resp.ConvertToAPIUser() + result, err := user_resp.ConvertToAPIUser() + assert.NoError(err) assert.Equal(int64(44067298), result.ID) assert.Equal(user_resp.Data.User.Result.Legacy.FollowersCount, result.FollowersCount) } diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index a594b40..e5662f4 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -1323,7 +1323,7 @@ func (api *API) GetHomeTimeline(cursor string, is_following_only bool) (TweetTro // Get User // -------- -func (api API) GetUser(handle UserHandle) (APIUser, error) { +func (api API) GetUser(handle UserHandle) (User, error) { url, err := url.Parse(GraphqlURL{ BaseUrl: "https://api.twitter.com/graphql/SAMkL5y_N9pmahSw8yy6gw/UserByScreenName", Variables: GraphqlVariables{ @@ -1362,7 +1362,26 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) { var response UserResponse err = api.do_http(url.String(), "", &response) - return response.ConvertToAPIUser(), err + if err != nil { + return User{}, err + } + apiUser, err := response.ConvertToAPIUser() + if errors.Is(err, ErrDoesntExist) { + return User{}, err + } + if apiUser.ScreenName == "" { + if apiUser.IsBanned || apiUser.DoesntExist { + ret := GetUnknownUserWithHandle(handle) + ret.IsBanned = apiUser.IsBanned + ret.IsDeleted = apiUser.DoesntExist + return ret, nil + } + apiUser.ScreenName = string(handle) + } + if err != nil { + return User{}, fmt.Errorf("Error fetching user %q:\n %w", handle, err) + } + return ParseSingleUser(apiUser) } // Paginated Search diff --git a/pkg/scraper/tweet_trove.go b/pkg/scraper/tweet_trove.go index ca11bc8..f3e514a 100644 --- a/pkg/scraper/tweet_trove.go +++ b/pkg/scraper/tweet_trove.go @@ -1,6 +1,7 @@ package scraper import ( + "errors" "fmt" "strings" @@ -96,7 +97,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() { if already_fetched { // If the user is already fetched and it's an intact user, don't fetch it again - if user.JoinDate.Unix() != (Timestamp{}).Unix() { + if user.JoinDate.Unix() != (Timestamp{}).Unix() && user.JoinDate.Unix() != 0 { log.Debugf("Skipping %q due to intact user", handle) continue } @@ -110,7 +111,10 @@ func (trove *TweetTrove) FetchTombstoneUsers() { log.Debug("Getting tombstone user: " + handle) user, err := GetUser(handle) - if err != nil { + if errors.Is(err, ErrDoesntExist) { + user = GetUnknownUserWithHandle(handle) + user.IsDeleted = true + } else if err != nil { panic(fmt.Errorf("Error getting tombstoned user with handle %q: \n %w", handle, err)) } diff --git a/pkg/scraper/user.go b/pkg/scraper/user.go index cc83606..1191b96 100644 --- a/pkg/scraper/user.go +++ b/pkg/scraper/user.go @@ -179,20 +179,7 @@ func GetUser(handle UserHandle) (User, error) { if err != nil { return User{}, err } - apiUser, err := session.GetUser(handle) - if apiUser.ScreenName == "" { - if apiUser.IsBanned || apiUser.DoesntExist { - ret := GetUnknownUserWithHandle(handle) - ret.IsBanned = apiUser.IsBanned - ret.IsDeleted = apiUser.DoesntExist - return ret, nil - } - apiUser.ScreenName = string(handle) - } - if err != nil { - return User{}, fmt.Errorf("Error fetching user %q:\n %w", handle, err) - } - return ParseSingleUser(apiUser) + return session.GetUser(handle) } /** diff --git a/pkg/scraper/user_test.go b/pkg/scraper/user_test.go index 6fe3742..38061e7 100644 --- a/pkg/scraper/user_test.go +++ b/pkg/scraper/user_test.go @@ -15,18 +15,20 @@ import ( func TestParseSingleUser(t *testing.T) { assert := assert.New(t) + require := require.New(t) data, err := os.ReadFile("test_responses/michael_malice_user_profile.json") if err != nil { panic(err) } var user_resp UserResponse err = json.Unmarshal(data, &user_resp) - require.NoError(t, err) + require.NoError(err) - apiUser := user_resp.ConvertToAPIUser() + apiUser, err := user_resp.ConvertToAPIUser() + require.NoError(err) user, err := ParseSingleUser(apiUser) - require.NoError(t, err) + require.NoError(err) assert.Equal(UserID(44067298), user.ID) assert.Equal("Michael Malice", user.DisplayName) @@ -62,7 +64,8 @@ func TestParseBannedUser(t *testing.T) { err = json.Unmarshal(data, &user_resp) require.NoError(t, err) - apiUser := user_resp.ConvertToAPIUser() + apiUser, err := user_resp.ConvertToAPIUser() + require.NoError(t, err) user, err := ParseSingleUser(apiUser) require.NoError(t, err) @@ -86,22 +89,9 @@ func TestParseDeletedUser(t *testing.T) { err = json.Unmarshal(data, &user_resp) require.NoError(t, err) - handle := "Some Random Deleted User" - - apiUser := user_resp.ConvertToAPIUser() - apiUser.ScreenName = string(handle) // This is done in scraper.GetUser, since users are retrieved by handle anyway - - user, err := ParseSingleUser(apiUser) - require.NoError(t, err) - assert.Equal(UserID(0), user.ID) - assert.True(user.IsIdFake) - assert.True(user.IsNeedingFakeID) - assert.Equal(user.Bio, "") - assert.Equal(user.Handle, UserHandle(handle)) - - // Test generation of profile images for deleted user - assert.Equal("https://abs.twimg.com/sticky/default_profile_images/default_profile.png", user.GetTinyProfileImageUrl()) - assert.Equal("default_profile.png", user.GetTinyProfileImageLocalPath()) + _, err = user_resp.ConvertToAPIUser() + assert.Error(err) + assert.ErrorIs(err, ErrDoesntExist) } /**