"ConvertToAPIUser() now returns an error indicating a "not-found" response, which propagates through "GetUser" API calls

cmd/twitter/main.go

@@ -288,7 +288,15 @@ func create_profile(target_dir string) {
  */
 func fetch_user(handle scraper.UserHandle) {
 	user, err := scraper.GetUser(handle)
-	if is_scrape_failure(err) {
+	if errors.Is(err, scraper.ErrDoesntExist) {
+		// There's several reasons we could get a ErrDoesntExist:
+		//   1. account never existed (user made a CLI typo)
+		//   2. user changed their handle
+		//   3. user deleted their account
+		// In case (1), we should just report the error; in case (2) and (3), it would be nice to rescrape by ID,
+		// but that feels kind of too complicated to do here.  So just report the error and let the user decide
+		die(fmt.Sprintf("User with handle %q doesn't exist.  Check spelling, or try scraping with the ID instead", handle), false, -1)
+	} else if is_scrape_failure(err) {
 		die(err.Error(), false, -1)
 	}
 	log.Debug(user)

internal/webserver/handler_login.go

@@ -70,7 +70,7 @@ func (app *Application) after_login(w http.ResponseWriter, r *http.Request, api
 
 	// Ensure the user is downloaded
 	user, err := scraper.GetUser(api.UserHandle)
-	if err != nil {
+	if err != nil { // ErrDoesntExist or otherwise
 		app.error_404(w, r)
 		return
 	}

internal/webserver/handler_user_feed.go

@@ -16,16 +16,18 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
 	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
 
 	user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[0]))
-	if err != nil {
+	if errors.Is(err, persistence.ErrNotInDatabase) {
 		if !app.IsScrapingDisabled {
 			user, err = scraper.GetUser(scraper.UserHandle(parts[0]))
 		}
-		if err != nil {
+		if err != nil { // ErrDoesntExist or otherwise
 			app.error_404(w, r)
 			return
 		}
 		panic_if(app.Profile.SaveUser(&user))
 		panic_if(app.Profile.DownloadUserContentFor(&user, &app.API))
+	} else if err != nil {
+		panic(err)
 	}
 
 	if len(parts) > 1 && parts[1] == "followers" {

pkg/scraper/api_types.go

@@ -295,6 +295,7 @@ type UserResponse struct {
 	Data struct {
 		User struct {
 			Result struct {
+				MetaTypename       string  `json:"__typename"`
 				ID                 int64   `json:"rest_id,string"`
 				Legacy             APIUser `json:"legacy"`
 				IsBlueVerified     bool    `json:"is_blue_verified"`
@@ -312,7 +313,12 @@ type UserResponse struct {
 	} `json:"errors"`
 }
 
-func (u UserResponse) ConvertToAPIUser() APIUser {
+func (u UserResponse) ConvertToAPIUser() (APIUser, error) {
+	if u.Data.User.Result.MetaTypename == "" {
+		// Completely empty response (user not found)
+		return APIUser{}, ErrDoesntExist
+	}
+
 	ret := u.Data.User.Result.Legacy
 	ret.ID = u.Data.User.Result.ID
 	ret.Verified = u.Data.User.Result.IsBlueVerified
@@ -338,7 +344,7 @@ func (u UserResponse) ConvertToAPIUser() APIUser {
 		ret.DoesntExist = true
 	}
 
-	return ret
+	return ret, nil
 }
 
 type Entry struct {

pkg/scraper/api_types_test.go

@@ -69,7 +69,8 @@ func TestUserProfileToAPIUser(t *testing.T) {
 	err = json.Unmarshal(data, &user_resp)
 	assert.NoError(err)
 
-	result := user_resp.ConvertToAPIUser()
+	result, err := user_resp.ConvertToAPIUser()
+	assert.NoError(err)
 	assert.Equal(int64(44067298), result.ID)
 	assert.Equal(user_resp.Data.User.Result.Legacy.FollowersCount, result.FollowersCount)
 }

pkg/scraper/api_types_v2.go

@@ -1323,7 +1323,7 @@ func (api *API) GetHomeTimeline(cursor string, is_following_only bool) (TweetTro
 // Get User
 // --------
 
-func (api API) GetUser(handle UserHandle) (APIUser, error) {
+func (api API) GetUser(handle UserHandle) (User, error) {
 	url, err := url.Parse(GraphqlURL{
 		BaseUrl: "https://api.twitter.com/graphql/SAMkL5y_N9pmahSw8yy6gw/UserByScreenName",
 		Variables: GraphqlVariables{
@@ -1362,7 +1362,26 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) {
 
 	var response UserResponse
 	err = api.do_http(url.String(), "", &response)
-	return response.ConvertToAPIUser(), err
+	if err != nil {
+		return User{}, err
+	}
+	apiUser, err := response.ConvertToAPIUser()
+	if errors.Is(err, ErrDoesntExist) {
+		return User{}, err
+	}
+	if apiUser.ScreenName == "" {
+		if apiUser.IsBanned || apiUser.DoesntExist {
+			ret := GetUnknownUserWithHandle(handle)
+			ret.IsBanned = apiUser.IsBanned
+			ret.IsDeleted = apiUser.DoesntExist
+			return ret, nil
+		}
+		apiUser.ScreenName = string(handle)
+	}
+	if err != nil {
+		return User{}, fmt.Errorf("Error fetching user %q:\n  %w", handle, err)
+	}
+	return ParseSingleUser(apiUser)
 }
 
 // Paginated Search

pkg/scraper/tweet_trove.go

@@ -1,6 +1,7 @@
 package scraper
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -96,7 +97,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() {
 
 		if already_fetched {
 			// If the user is already fetched and it's an intact user, don't fetch it again
-			if user.JoinDate.Unix() != (Timestamp{}).Unix() {
+			if user.JoinDate.Unix() != (Timestamp{}).Unix() && user.JoinDate.Unix() != 0 {
 				log.Debugf("Skipping %q due to intact user", handle)
 				continue
 			}
@@ -110,7 +111,10 @@ func (trove *TweetTrove) FetchTombstoneUsers() {
 
 		log.Debug("Getting tombstone user: " + handle)
 		user, err := GetUser(handle)
-		if err != nil {
+		if errors.Is(err, ErrDoesntExist) {
+			user = GetUnknownUserWithHandle(handle)
+			user.IsDeleted = true
+		} else if err != nil {
 			panic(fmt.Errorf("Error getting tombstoned user with handle %q: \n  %w", handle, err))
 		}
 

pkg/scraper/user.go

@@ -179,20 +179,7 @@ func GetUser(handle UserHandle) (User, error) {
 	if err != nil {
 		return User{}, err
 	}
-	apiUser, err := session.GetUser(handle)
+	return session.GetUser(handle)
-	if apiUser.ScreenName == "" {
-		if apiUser.IsBanned || apiUser.DoesntExist {
-			ret := GetUnknownUserWithHandle(handle)
-			ret.IsBanned = apiUser.IsBanned
-			ret.IsDeleted = apiUser.DoesntExist
-			return ret, nil
-		}
-		apiUser.ScreenName = string(handle)
-	}
-	if err != nil {
-		return User{}, fmt.Errorf("Error fetching user %q:\n  %w", handle, err)
-	}
-	return ParseSingleUser(apiUser)
 }
 
 /**

pkg/scraper/user_test.go

@@ -15,18 +15,20 @@ import (
 
 func TestParseSingleUser(t *testing.T) {
 	assert := assert.New(t)
+	require := require.New(t)
 	data, err := os.ReadFile("test_responses/michael_malice_user_profile.json")
 	if err != nil {
 		panic(err)
 	}
 	var user_resp UserResponse
 	err = json.Unmarshal(data, &user_resp)
-	require.NoError(t, err)
+	require.NoError(err)
 
-	apiUser := user_resp.ConvertToAPIUser()
+	apiUser, err := user_resp.ConvertToAPIUser()
+	require.NoError(err)
 
 	user, err := ParseSingleUser(apiUser)
-	require.NoError(t, err)
+	require.NoError(err)
 
 	assert.Equal(UserID(44067298), user.ID)
 	assert.Equal("Michael Malice", user.DisplayName)
@@ -62,7 +64,8 @@ func TestParseBannedUser(t *testing.T) {
 	err = json.Unmarshal(data, &user_resp)
 	require.NoError(t, err)
 
-	apiUser := user_resp.ConvertToAPIUser()
+	apiUser, err := user_resp.ConvertToAPIUser()
+	require.NoError(t, err)
 
 	user, err := ParseSingleUser(apiUser)
 	require.NoError(t, err)
@@ -86,22 +89,9 @@ func TestParseDeletedUser(t *testing.T) {
 	err = json.Unmarshal(data, &user_resp)
 	require.NoError(t, err)
 
-	handle := "Some Random Deleted User"
+	_, err = user_resp.ConvertToAPIUser()
-
+	assert.Error(err)
-	apiUser := user_resp.ConvertToAPIUser()
+	assert.ErrorIs(err, ErrDoesntExist)
-	apiUser.ScreenName = string(handle) // This is done in scraper.GetUser, since users are retrieved by handle anyway
-
-	user, err := ParseSingleUser(apiUser)
-	require.NoError(t, err)
-	assert.Equal(UserID(0), user.ID)
-	assert.True(user.IsIdFake)
-	assert.True(user.IsNeedingFakeID)
-	assert.Equal(user.Bio, "<blank>")
-	assert.Equal(user.Handle, UserHandle(handle))
-
-	// Test generation of profile images for deleted user
-	assert.Equal("https://abs.twimg.com/sticky/default_profile_images/default_profile.png", user.GetTinyProfileImageUrl())
-	assert.Equal("default_profile.png", user.GetTinyProfileImageLocalPath())
 }
 
 /**