me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scraper requests now report invalidated or expired sessions

Browse Source
This commit is contained in:
Alessio 2024-08-18 16:22:37 -07:00
parent 8321d65f17
commit 91f722b7fa
2 changed files with 36 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/scraper/api_errors.go
View File

@ -5,14 +5,15 @@ import (
)
var (
END_OF_FEED = errors.New("End of feed")
ErrDoesntExist = errors.New("Doesn't exist")
EXTERNAL_API_ERROR = errors.New("Unexpected result from external API")
ErrorIsTombstone = errors.New("tweet is a tombstone")
ErrRateLimited = errors.New("rate limited")
ErrorDMCA = errors.New("video is DMCAed, unable to download (HTTP 403 Forbidden)")
ErrMediaDownload404 = errors.New("media download HTTP 404")
ErrLoginRequired = errors.New("login required; please provide `--session <user>` flag")
END_OF_FEED = errors.New("End of feed")
ErrDoesntExist = errors.New("Doesn't exist")
EXTERNAL_API_ERROR = errors.New("Unexpected result from external API")
ErrorIsTombstone = errors.New("tweet is a tombstone")
ErrRateLimited = errors.New("rate limited")
ErrorDMCA = errors.New("video is DMCAed, unable to download (HTTP 403 Forbidden)")
ErrMediaDownload404 = errors.New("media download HTTP 404")
ErrLoginRequired = errors.New("login required; please provide `--session <user>` flag")
ErrSessionInvalidated = errors.New("session invalidated by Twitter")
// These are not API errors, but network errors generally
ErrNoInternet = errors.New("no internet connection")

27
pkg/scraper/api_request_utils.go
View File

@ -155,6 +155,21 @@ func is_timeout(err error) bool {
return false
}
func is_session_invalidated(respBody []byte) bool {
var result struct {
Errors []struct {
Message string
Code int
} `json:"errors"`
}
err := json.Unmarshal(respBody, &result)
if err != nil {
panic(err)
}
return len(result.Errors) == 1 &&
(result.Errors[0].Message == "Could not authenticate you" || result.Errors[0].Code == 32)
}
func (api *API) do_http_POST(remote_url string, body string, result interface{}) error {
req, err := http.NewRequest("POST", remote_url, strings.NewReader(body))
if err != nil {
@ -198,6 +213,10 @@ func (api *API) do_http_POST(remote_url string, body string, result interface{})
}
if resp.StatusCode != 200 {
if resp.StatusCode == 401 && is_session_invalidated(respBody) {
return ErrSessionInvalidated
}
responseHeaders := ""
for header := range resp.Header {
responseHeaders += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header))
@ -260,6 +279,10 @@ func (api *API) do_http(remote_url string, cursor string, result interface{}) er
}
if resp.StatusCode != 200 && resp.StatusCode != 403 {
if resp.StatusCode == 401 && is_session_invalidated(body) {
return ErrSessionInvalidated
}
responseHeaders := ""
for header := range resp.Header {
responseHeaders += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header))
@ -365,6 +388,10 @@ func (api *API) DownloadMedia(remote_url string) ([]byte, error) {
}
print_curl_cmd(*req, api.Client.Jar.Cookies(url))
if resp.StatusCode == 401 && is_session_invalidated(body) {
return body, ErrSessionInvalidated
}
responseHeaders := ""
for header := range resp.Header {
responseHeaders += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header))