diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 588d6d5..12d67df 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -129,7 +129,12 @@ func main() { scraper.InitApi(profile.LoadSession(scraper.UserHandle(*session_name))) // fmt.Printf("Operating as user: @%s\n", scraper.the_api.UserHandle) } else { - scraper.InitApi(scraper.NewGuestSession()) + session, err := scraper.NewGuestSession() + if err != nil { + log.Warnf("Unable to initialize guest session! Might be a network issue") + } else { + scraper.InitApi(session) + } } switch operation { @@ -222,7 +227,10 @@ func main() { // - password: twitter account password func login(username string, password string) { // Skip the scraper.the_api variable, just use a local one since no scraping is happening - api := scraper.NewGuestSession() + api, err := scraper.NewGuestSession() + if err != nil { + die(fmt.Sprintf("Unable to create session: %s", err.Error()), false, 1) + } challenge := api.LogIn(username, password) if challenge != nil { fmt.Printf("Secondary challenge issued:\n") diff --git a/internal/webserver/handler_login.go b/internal/webserver/handler_login.go index 074d3e2..f5f37e4 100644 --- a/internal/webserver/handler_login.go +++ b/internal/webserver/handler_login.go @@ -42,7 +42,10 @@ func (app *Application) Login(w http.ResponseWriter, r *http.Request) { panic_if(json.Unmarshal(data, &form)) // TODO: HTTP 400 not 500 form.Validate() if len(form.FormErrors) == 0 { - api := scraper.NewGuestSession() + api, err := scraper.NewGuestSession() + if err != nil { + panic(err.Error()) // Return it as a toast + } challenge := api.LogIn(form.Username, form.Password) if challenge != nil { panic( // Middleware will trap this panic and return an HTMX error toast diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 96ba0f7..345f650 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -62,7 +62,6 @@ func (app *Application) WithMiddlewares() http.Handler { func (app *Application) SetActiveUser(handle scraper.UserHandle) error { if handle == "no account" { - scraper.InitApi(scraper.NewGuestSession()) app.ActiveUser = get_default_user() app.IsScrapingDisabled = true // API requests will fail b/c not logged in } else { diff --git a/pkg/scraper/api_errors.go b/pkg/scraper/api_errors.go index 1d2052a..ff7d717 100644 --- a/pkg/scraper/api_errors.go +++ b/pkg/scraper/api_errors.go @@ -11,5 +11,8 @@ var ( ErrorIsTombstone = errors.New("tweet is a tombstone") ErrRateLimited = errors.New("rate limited") ErrorDMCA = errors.New("video is DMCAed, unable to download (HTTP 403 Forbidden)") - ErrRequestTimeout = errors.New("request timed out") + + // These are not API errors, but network errors generally + ErrNoInternet = errors.New("no internet connection") + ErrRequestTimeout = errors.New("request timed out") ) diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go index 2f716cb..2c18a5d 100644 --- a/pkg/scraper/api_request_utils.go +++ b/pkg/scraper/api_request_utils.go @@ -111,10 +111,10 @@ func (api API) add_authentication_headers(req *http.Request) { } } -func NewGuestSession() API { - guestAPIString, err := GetGuestToken() +func NewGuestSession() (API, error) { + guestAPIString, err := GetGuestTokenWithRetries(3, 1*time.Second) if err != nil { - panic(err) + return API{}, err } jar, err := cookiejar.New(nil) @@ -129,7 +129,7 @@ func NewGuestSession() API { Jar: jar, }, CSRFToken: "", - } + }, nil } func (api *API) update_csrf_token() { diff --git a/pkg/scraper/guest_token.go b/pkg/scraper/guest_token.go index 1095307..2a7c243 100644 --- a/pkg/scraper/guest_token.go +++ b/pkg/scraper/guest_token.go @@ -2,8 +2,11 @@ package scraper import ( "encoding/json" + "errors" "fmt" "io" + "log" + "net" "net/http" "time" ) @@ -15,7 +18,20 @@ type GuestTokenResponse struct { var guestToken GuestTokenResponse +func GetGuestTokenWithRetries(n int, sleep time.Duration) (ret string, err error) { + for i := 0; i < n; i++ { + ret, err = GetGuestToken() + if err == nil { + return + } + log.Printf("Failed to get guest token: %s\nRetrying...", err.Error()) + time.Sleep(sleep) + } + return +} + func GetGuestToken() (string, error) { + // Guest token is still valid; no need for new one if time.Since(guestToken.RefreshedAt).Hours() < 1 { return guestToken.Token, nil } @@ -29,6 +45,11 @@ func GetGuestToken() (string, error) { resp, err := client.Do(req) if err != nil { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && dnsErr.Err == "server misbehaving" && dnsErr.Temporary() { + return "", ErrNoInternet + } + return "", fmt.Errorf("Error executing HTTP request:\n %w", err) } diff --git a/pkg/scraper/user.go b/pkg/scraper/user.go index e8fad20..cc83606 100644 --- a/pkg/scraper/user.go +++ b/pkg/scraper/user.go @@ -175,7 +175,11 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) { // Calls API#GetUser and returns the parsed result func GetUser(handle UserHandle) (User, error) { - apiUser, err := NewGuestSession().GetUser(handle) + session, err := NewGuestSession() // This endpoint works better if you're not logged in + if err != nil { + return User{}, err + } + apiUser, err := session.GetUser(handle) if apiUser.ScreenName == "" { if apiUser.IsBanned || apiUser.DoesntExist { ret := GetUnknownUserWithHandle(handle)