Add support for TweetWithVisibilityResults API type, which has a different JSON structure for some reason

scraper/api_types_v2.go

@@ -133,8 +133,7 @@ func (u APIV2UserResult) ToUser() User {
 	return user
 }
 
-type APIV2Result struct {
+type _Result struct {
-	Result struct {
 	ID int64 `json:"rest_id,string"`
 	Legacy APIV2Tweet `json:"legacy"`
 	Tombstone *struct {
@@ -145,21 +144,35 @@ type APIV2Result struct {
 	Core *APIV2UserResult `json:"core"`
 	Card APIV2Card `json:"card"`
 	QuotedStatusResult *APIV2Result `json:"quoted_status_result"`
+}
+
+type APIV2Result struct {
+	Result struct {
+		_Result
+		Tweet _Result `json:"tweet"`
 	} `json:"result"`
 }
 func (api_result APIV2Result) ToTweetTrove() TweetTrove {
 	ret := NewTweetTrove()
 
-	if api_result.Result.Core != nil {
+	if api_result.Result.Legacy.ID == 0 && api_result.Result.Tweet.Legacy.ID != 0 {
-		main_user := api_result.Result.Core.ToUser()
+		// If the tweet has "__typename" of "TweetWithVisibilityResults", it uses a new structure with
-		ret.Users[main_user.ID] = main_user
+		// a "tweet" field with the regular data, alongside a "tweetInterstitial" field which is ignored
-	} /*else {
+		// for now.
-		// TODO
+		log.Debug("Using Inner Tweet")
-	}*/
+		api_result.Result._Result = api_result.Result.Tweet
+	}
 
 	main_tweet_trove := api_result.Result.Legacy.ToTweetTrove()
 	ret.MergeWith(main_tweet_trove)
 
+	// Parse the User info
+	if api_result.Result.Core != nil {
+		// `Core` is nil for tombstones because they don't carry user info.  Nothing to do here
+		main_user := api_result.Result.Core.ToUser()
+		ret.Users[main_user.ID] = main_user
+	}
+
 	// Handle quoted tweet
 	if api_result.Result.QuotedStatusResult != nil {
 		quoted_api_result := api_result.Result.QuotedStatusResult

scraper/api_types_v2_test.go

@@ -539,3 +539,30 @@ func TestAPIV2GetMainInstructionFromFeed(t *testing.T) {
 	assert.Equal(len(feed.GetMainInstruction().Entries), 42)
 	assert.Equal(feed.GetMainInstruction().Entries[41].EntryID, "asdf")
 }
+
+/**
+ * Should handle an entry in the feed that's just a tombstone
+ */
+func TestAPIV2TombstoneEntry(t *testing.T) {
+
+}
+
+
+func TestTweetWithWarning(t *testing.T) {
+	assert := assert.New(t)
+	data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_warning.json")
+	if err != nil {
+		panic(err)
+	}
+	var tweet_result APIV2Result
+	err = json.Unmarshal(data, &tweet_result)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	trove := tweet_result.ToTweetTrove()
+
+	assert.Len(trove.Retweets, 1)
+	assert.Len(trove.Tweets, 2)
+	assert.Len(trove.Users, 3)
+}