me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save tweets with unknown user as <UNKNOWN USER> instead of UserID 0, which causes foreign key errors

Browse Source
This commit is contained in:
Alessio 2024-06-10 15:47:13 -07:00
parent 4dbe8b1558
commit cac87ec973
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/scraper/api_types_v2.go
View File

@ -522,7 +522,7 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
} }
ret, err := e.Content.ItemContent.TweetResults.ToTweetTrove() ret, err := e.Content.ItemContent.TweetResults.ToTweetTrove()
// Handle tombstones in parent reply thread // Handle tombstones in the focused tweet and parent reply thread
if errors.Is(err, ErrorIsTombstone) { if errors.Is(err, ErrorIsTombstone) {
ret = NewTweetTrove() // clear the result just in case there is a TweetID(0) in it ret = NewTweetTrove() // clear the result just in case there is a TweetID(0) in it
tombstoned_tweet := APITweet{} tombstoned_tweet := APITweet{}
@ -547,6 +547,10 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
if err != nil { if err != nil {
panic(err) panic(err)
} }
fake_user := GetUnknownUser()
ret.Users[fake_user.ID] = fake_user
parsed_tombstone_tweet.UserID = fake_user.ID
ret.Tweets[parsed_tombstone_tweet.ID] = parsed_tombstone_tweet ret.Tweets[parsed_tombstone_tweet.ID] = parsed_tombstone_tweet
} else if err != nil { } else if err != nil {
if e.Content.ItemContent.APIV2UserResult.UserResults.Result.ID != 0 { if e.Content.ItemContent.APIV2UserResult.UserResults.Result.ID != 0 {
@ -764,9 +768,9 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
// Fill out the replied tweet's UserID using this tweet's "in_reply_to_user_id". // Fill out the replied tweet's UserID using this tweet's "in_reply_to_user_id".
// If this tweet doesn't have it (i.e., this tweet is also a tombstone), create a fake user instead, and add it to the tweet trove. // If this tweet doesn't have it (i.e., this tweet is also a tombstone), create a fake user instead, and add it to the tweet trove.
if replied_tweet.UserID == 0 { if replied_tweet.UserID == 0 || replied_tweet.UserID == GetUnknownUser().ID {
replied_tweet.UserID = tweet.in_reply_to_user_id replied_tweet.UserID = tweet.in_reply_to_user_id
if replied_tweet.UserID == 0 { if replied_tweet.UserID == 0 || replied_tweet.UserID == GetUnknownUser().ID {
fake_user := GetUnknownUser() fake_user := GetUnknownUser()
ret.Users[fake_user.ID] = fake_user ret.Users[fake_user.ID] = fake_user
replied_tweet.UserID = fake_user.ID replied_tweet.UserID = fake_user.ID

11
pkg/scraper/api_types_v2_test.go
View File

@ -584,12 +584,21 @@ func TestAPIV2TombstoneEntry(t *testing.T) {
trove := entry.ToTweetTrove() trove := entry.ToTweetTrove()
assert.NoError(err) assert.NoError(err)
assert.Len(trove.Tweets, 1) assert.Len(trove.Tweets, 1)
assert.Len(trove.Users, 0) assert.Len(trove.Users, 1)
assert.Len(trove.Retweets, 0) assert.Len(trove.Retweets, 0)
tweet, is_ok := trove.Tweets[1454515503242829830] tweet, is_ok := trove.Tweets[1454515503242829830]
assert.True(is_ok) assert.True(is_ok)
assert.True(tweet.IsStub)
assert.Equal("hidden", tweet.TombstoneType)
assert.Equal(tweet.ID, TweetID(1454515503242829830)) assert.Equal(tweet.ID, TweetID(1454515503242829830))
expected_user := GetUnknownUser()
assert.Equal(expected_user.ID, tweet.UserID)
assert.Len(trove.Users, 1)
_, is_ok = trove.Users[expected_user.ID]
assert.True(is_ok)
} }
func TestAPIV2UserFeedTombstoneEntry(t *testing.T) { func TestAPIV2UserFeedTombstoneEntry(t *testing.T) {