me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

When doing reply-joining of tombstones, ignore cursors and labels

Browse Source
This commit is contained in:
Alessio 2023-06-21 13:49:46 -03:00
parent 3dc9889725
commit eae3ebe3e7
1 changed files with 31 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scraper/api_types_v2.go
View File

@ -587,17 +587,17 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
continue continue
} }
// Infer "in_reply_to_id" for tombstoned tweets from the order of entries, if applicable // Infer "in_reply_to_id" for tombstoned tweets from the order of entries, if applicable
if entry.Content.EntryType != "TimelineTimelineItem" { if entry.Content.EntryType == "TimelineTimelineItem" {
// Only check back up the parent thread, which will all be "TimelineTimelineItems". entry_type, main_tweet_id := entry.ParseID()
// i.e., skip replies if entry_type == "cursor-showmorethreadsprompt" || entry_type == "cursor-bottom" || entry_type == "cursor-showmorethreads" {
// TODO: maybe don't skip replies? // Skip cursors
// - "cursor-bottom" => auto-loads more replies when you scroll it into view
// - "cursor-showmorethreadsprompt" => "Show additional replies, including those that may contain offensive content" button
// - "cursor-showmorethreads" => "Show more replies" button
continue continue
} }
entry_type, main_tweet_id := entry.ParseID() if entry_type == "label" {
if entry_type == "cursor-showmorethreadsprompt" || entry_type == "cursor-bottom" { // Skip labels / headers
// Skip cursors
// - "cursor-bottom" => load more high-quality replies
// - "cursor-showmorethreadsprompt" => "Show additional replies, including those that may contain offensive content"
continue continue
} }
if entry_type != "tweet" { if entry_type != "tweet" {
@ -616,6 +616,9 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
_, prev_entry_id := api_response.GetMainInstruction().Entries[i-1].ParseID() _, prev_entry_id := api_response.GetMainInstruction().Entries[i-1].ParseID()
main_tweet.InReplyToID = prev_entry_id main_tweet.InReplyToID = prev_entry_id
ret.Tweets[main_tweet_id] = main_tweet ret.Tweets[main_tweet_id] = main_tweet
} else if entry.Content.EntryType == "TimelineTimelineModule" {
// TODO: check reply threads for tombstones as well
}
} }
// Add in any tombstoned user handles and IDs if possible, by reading from the replies // Add in any tombstoned user handles and IDs if possible, by reading from the replies