me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for TweetWithVisibilityResults API type, which has a different JSON structure for some reason

Browse Source
This commit is contained in:
Alessio 2022-02-28 13:44:42 -08:00
parent 648098352b
commit 0909efb18b
2 changed files with 56 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
scraper/api_types_v2.go
View File

@ -133,33 +133,46 @@ func (u APIV2UserResult) ToUser() User {
return user
}
type _Result struct {
ID int64 `json:"rest_id,string"`
Legacy APIV2Tweet `json:"legacy"`
Tombstone *struct {
Text struct {
Text string `json:"text"`
} `json:"text"`
} `json:"tombstone"`
Core *APIV2UserResult `json:"core"`
Card APIV2Card `json:"card"`
QuotedStatusResult *APIV2Result `json:"quoted_status_result"`
}
type APIV2Result struct {
Result struct {
ID int64 `json:"rest_id,string"`
Legacy APIV2Tweet `json:"legacy"`
Tombstone *struct {
Text struct {
Text string `json:"text"`
} `json:"text"`
} `json:"tombstone"`
Core *APIV2UserResult `json:"core"`
Card APIV2Card `json:"card"`
QuotedStatusResult *APIV2Result `json:"quoted_status_result"`
_Result
Tweet _Result `json:"tweet"`
} `json:"result"`
}
func (api_result APIV2Result) ToTweetTrove() TweetTrove {
ret := NewTweetTrove()
if api_result.Result.Core != nil {
main_user := api_result.Result.Core.ToUser()
ret.Users[main_user.ID] = main_user
} /*else {
// TODO
}*/
if api_result.Result.Legacy.ID == 0 && api_result.Result.Tweet.Legacy.ID != 0 {
// If the tweet has "__typename" of "TweetWithVisibilityResults", it uses a new structure with
// a "tweet" field with the regular data, alongside a "tweetInterstitial" field which is ignored
// for now.
log.Debug("Using Inner Tweet")
api_result.Result._Result = api_result.Result.Tweet
}
main_tweet_trove := api_result.Result.Legacy.ToTweetTrove()
ret.MergeWith(main_tweet_trove)
// Parse the User info
if api_result.Result.Core != nil {
// `Core` is nil for tombstones because they don't carry user info. Nothing to do here
main_user := api_result.Result.Core.ToUser()
ret.Users[main_user.ID] = main_user
}
// Handle quoted tweet
if api_result.Result.QuotedStatusResult != nil {
quoted_api_result := api_result.Result.QuotedStatusResult

27
scraper/api_types_v2_test.go
View File

@ -539,3 +539,30 @@ func TestAPIV2GetMainInstructionFromFeed(t *testing.T) {
assert.Equal(len(feed.GetMainInstruction().Entries), 42)
assert.Equal(feed.GetMainInstruction().Entries[41].EntryID, "asdf")
}
/**
* Should handle an entry in the feed that's just a tombstone
*/
func TestAPIV2TombstoneEntry(t *testing.T) {
}
func TestTweetWithWarning(t *testing.T) {
assert := assert.New(t)
data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_warning.json")
if err != nil {
panic(err)
}
var tweet_result APIV2Result
err = json.Unmarshal(data, &tweet_result)
if err != nil {
t.Errorf(err.Error())
}
trove := tweet_result.ToTweetTrove()
assert.Len(trove.Retweets, 1)
assert.Len(trove.Tweets, 2)
assert.Len(trove.Users, 3)
}