me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix detection of deleted tweets

Browse Source
This commit is contained in:
Alessio 2023-10-23 13:48:38 -03:00
parent 1fd41c4c32
commit 9448a84b28
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/scraper/api_types_v2.go
View File

@ -12,6 +12,7 @@ import (
) )
var ErrorIsTombstone = errors.New("tweet is a tombstone") var ErrorIsTombstone = errors.New("tweet is a tombstone")
var ErrTweetNotFound = errors.New("api responded 'no status found with that ID'")
type CardValue struct { type CardValue struct {
Type string `json:"type"` Type string `json:"type"`
@ -590,6 +591,10 @@ type APIV2Response struct {
} `json:"search_timeline"` } `json:"search_timeline"`
} `json:"search_by_raw_query"` } `json:"search_by_raw_query"`
} `json:"data"` } `json:"data"`
Errors []struct {
Message string `json:"message"`
Code int `json:"code"`
} `json:"errors"`
} }
func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction { func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction {
@ -913,6 +918,13 @@ func (api *API) GetTweetDetail(tweet_id TweetID, cursor string) (APIV2Response,
} }
func (api *API) GetMoreTweetReplies(tweet_id TweetID, response *APIV2Response, min_tweets int) error { func (api *API) GetMoreTweetReplies(tweet_id TweetID, response *APIV2Response, min_tweets int) error {
if len(response.Errors) != 0 {
if response.Errors[0].Message == "_Missing: No status found with that ID." {
return ErrTweetNotFound
}
panic(fmt.Sprintf("Unknown error: %s", response.Errors[0].Message))
}
last_response := response last_response := response
for last_response.GetCursorBottom() != "" && len(response.GetMainInstruction().Entries) < min_tweets { for last_response.GetCursorBottom() != "" && len(response.GetMainInstruction().Entries) < min_tweets {
fresh_response, err := api.GetTweetDetail(tweet_id, last_response.GetCursorBottom()) fresh_response, err := api.GetTweetDetail(tweet_id, last_response.GetCursorBottom())

5
pkg/scraper/tweet.go
View File

@ -334,6 +334,11 @@ func GetTweetFullAPIV2(id TweetID, how_many int) (trove TweetTrove, err error) {
return return
} }
err = the_api.GetMoreTweetReplies(id, &resp, how_many) err = the_api.GetMoreTweetReplies(id, &resp, how_many)
if errors.Is(err, ErrTweetNotFound) {
trove := NewTweetTrove()
trove.Tweets[id] = Tweet{ID: id, TombstoneType: "deleted", IsConversationScraped: true}
return trove, nil
}
if err != nil && !errors.Is(err, END_OF_FEED) { if err != nil && !errors.Is(err, END_OF_FEED) {
err = fmt.Errorf("Error getting more replies in tweet detail: %d\n %w", id, err) err = fmt.Errorf("Error getting more replies in tweet detail: %d\n %w", id, err)
return return