me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix cursor bug in authenticated sessions

Browse Source
This commit is contained in:
Alessio 2023-06-03 07:51:39 -03:00
parent af93f44ed2
commit dea37c7556
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tests.sh
View File

@ -319,7 +319,7 @@ tw --session Offline_Twatter.session list_followed > /dev/null # Dummy operatio
# Test search # Test search
tw --session Offline_Twatter -n 1 search "from:michaelmalice constitution" # TODO: remove `-n 1` once the authenticated cursor bug is fixed tw --session Offline_Twatter search "from:michaelmalice constitution"
test $(sqlite3 twitter.db "select count(*) from tweets where user_id = 44067298 and text like '%constitution%'") -gt "30" # Not sure exactly how many test $(sqlite3 twitter.db "select count(*) from tweets where user_id = 44067298 and text like '%constitution%'") -gt "30" # Not sure exactly how many

8
scraper/api_request_utils.go
View File

@ -396,7 +396,7 @@ func (api API) GetSpace(id SpaceID) (SpaceResponse, error) {
return result, err return result, err
} }
func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) { func (api *API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
url, err := url.Parse(fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id)) url, err := url.Parse(fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id))
if err != nil { if err != nil {
panic(err) panic(err)
@ -414,7 +414,7 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
} }
// Resend the request to get more replies if necessary // Resend the request to get more replies if necessary
func (api API) GetMoreReplies(tweet_id TweetID, response *TweetResponse, max_replies int) error { func (api *API) GetMoreReplies(tweet_id TweetID, response *TweetResponse, max_replies int) error {
last_response := response last_response := response
for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_replies { for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_replies {
fresh_response, err := api.GetTweet(tweet_id, last_response.GetCursor()) fresh_response, err := api.GetTweet(tweet_id, last_response.GetCursor())
@ -466,7 +466,7 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) {
return result.ConvertToAPIUser(), err return result.ConvertToAPIUser(), err
} }
func (api API) Search(query string, cursor string) (TweetResponse, error) { func (api *API) Search(query string, cursor string) (TweetResponse, error) {
url, err := url.Parse("https://twitter.com/i/api/2/search/adaptive.json") url, err := url.Parse("https://twitter.com/i/api/2/search/adaptive.json")
if err != nil { if err != nil {
panic(err) panic(err)
@ -487,7 +487,7 @@ func (api API) Search(query string, cursor string) (TweetResponse, error) {
return result, err return result, err
} }
func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, max_results int) error { func (api *API) GetMoreTweetsFromSearch(query string, response *TweetResponse, max_results int) error {
last_response := response last_response := response
for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_results { for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_results {
fresh_response, err := api.Search(query, last_response.GetCursor()) fresh_response, err := api.Search(query, last_response.GetCursor())