diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index cbb0fe5..1c04222 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -454,10 +454,10 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove { } else if e.Content.EntryType == "TimelineTimelineItem" { ret, err := e.Content.ItemContent.TweetResults.ToTweetTrove() + // Handle tombstones in parent reply thread if errors.Is(err, ErrorIsTombstone) { - // Handle tombstones - ret = NewTweetTrove() // clear the result just in case - tombstoned_tweet := e.Content.ItemContent.TweetResults.Result.Legacy.APITweet // Will be empty to start + ret = NewTweetTrove() // clear the result just in case there is a TweetID(0) in it + tombstoned_tweet := APITweet{} // Capture the tombstone text var is_ok bool @@ -480,6 +480,8 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove { panic(err) } ret.Tweets[parsed_tombstone_tweet.ID] = parsed_tombstone_tweet + } else if err != nil { + panic(err) } return ret } @@ -626,11 +628,14 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { panic(fmt.Sprintf("Tombstoned tweet has no ID (should be %d)", tweet.InReplyToID)) } + // Fill out the replied tweet's UserID using this tweet's "in_reply_to_user_id". + // If this tweet doesn't have it (i.e., this tweet is also a tombstone), create a fake user instead, and add it to the tweet trove. if replied_tweet.UserID == 0 { replied_tweet.UserID = tweet.in_reply_to_user_id - if replied_tweet.UserID == 0 { // Still?? - log.Warn(fmt.Sprintf("Still couldn't find user for replied tweet %d", tweet.InReplyToID)) - continue + if replied_tweet.UserID == 0 { + fake_user := GetUnknownUser() + ret.Users[fake_user.ID] = fake_user + replied_tweet.UserID = fake_user.ID } } // replied_tweet.UserID should now be a real UserID diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index e1093a7..edf4bf1 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -601,12 +601,21 @@ func TestAPIV2ConversationThreadWithTombstones(t *testing.T) { assert.True(is_ok) assert.True(t1.IsStub) assert.Equal(TweetID(0), t1.InReplyToID) - // TODO: assert associated user is fake + assert.NotEqual(UserID(0), t1.UserID) + t1_user, is_ok := trove.Users[t1.UserID] + assert.True(is_ok) + assert.True(t1_user.IsIdFake) + assert.Equal(UserHandle(""), t1_user.Handle) t2, is_ok := trove.Tweets[1454521424144654344] assert.True(is_ok) assert.True(t2.IsStub) assert.Equal(TweetID(1454515503242829830), t2.InReplyToID) + assert.NotEqual(UserID(0), t2.UserID) + t2_user, is_ok := trove.Users[t2.UserID] + assert.True(is_ok) + assert.True(t2_user.IsIdFake) + assert.Equal(UserHandle(""), t2_user.Handle) t3, is_ok := trove.Tweets[1454522147750260742] assert.True(is_ok) diff --git a/scraper/user.go b/scraper/user.go index 0e340f2..1834cd2 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -84,6 +84,24 @@ Joined %s return ret } +func GetUnknownUser() User { + return User{ + ID: UserID(0x4000000000000000), // 2^62 + DisplayName: "", + Handle: UserHandle(""), + Bio: "", + FollowersCount: 0, + FollowingCount: 0, + Location: "", + Website: "", + JoinDate: TimestampFromUnix(0), + IsVerified: false, + IsPrivate: false, + IsNeedingFakeID: false, + IsIdFake: true, + } +} + /** * Unknown Users with handles are only created by direct GetUser calls (either `twitter fetch_user` * subcommand or as part of tombstone user fetching.)