API V2: parse User result

scraper/api_types_v2.go

@@ -10,6 +10,23 @@ import (
 	"strings"
 )
 
+type APIV2UserResult struct {
+	UserResults struct {
+		Result struct {
+			ID     int64   `json:"rest_id,string"`
+			Legacy APIUser `json:"legacy"`
+		} `json:"result"`
+	} `json:"user_results"`
+}
+func (u APIV2UserResult) ToUser() User {
+	user, err := ParseSingleUser(u.UserResults.Result.Legacy)
+	if err != nil {
+		panic(err)
+	}
+	user.ID = UserID(u.UserResults.Result.ID)
+	return user
+}
+
 type APIV2Tweet struct {
 	APITweet
 	RetweetedStatusResult struct {

scraper/api_types_v2_test.go

@@ -5,17 +5,53 @@ import (
 	"io/ioutil"
 	"encoding/json"
 
-	"offline_twitter/scraper"
+	. "offline_twitter/scraper"
+	"github.com/stretchr/testify/assert"
 )
 
-
-// Check a plain old tweet
-func TestAPIV2FeedSimpleTweet(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/feeds_api_v2/feed_simple_tweet.json")
+/**
+ * Parse an  APIV2User
+ */
+func TestAPIV2ParseUser(t *testing.T) {
+	data, err := ioutil.ReadFile("test_responses/api_v2/user_michael_malice.json")
 	if err != nil {
 		panic(err)
 	}
-	var feed scraper.APIV2Response
+
+	assert := assert.New(t)
+
+	var user_result APIV2UserResult
+	err = json.Unmarshal(data, &user_result)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	user := user_result.ToUser()
+
+	assert.Equal(user.ID, UserID(44067298))
+	assert.Equal(user.DisplayName, "Michael Malice")
+	assert.Equal(user.Handle, UserHandle("michaelmalice"))
+	assert.Equal(user.Bio, "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model")
+	assert.Equal(user.FollowingCount, 964)
+	assert.Equal(user.FollowersCount, 334571)
+	assert.Equal(user.Location, "Austin")
+	assert.Equal(user.Website, "https://amzn.to/3oInafv")
+	assert.Equal(user.JoinDate.Unix(), int64(1243920952))
+	assert.Equal(user.IsPrivate, false)
+	assert.Equal(user.IsVerified, true)
+	assert.Equal(user.IsBanned, false)
+	assert.Equal(user.ProfileImageUrl, "https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8.jpg")
+	assert.Equal(user.BannerImageUrl, "https://pbs.twimg.com/profile_banners/44067298/1615134676")
+	assert.Equal(user.PinnedTweetID, TweetID(1477347403023982596))
+}
+
+// Check a plain old tweet
+func TestAPIV2FeedSimpleTweet(t *testing.T) {
+	data, err := ioutil.ReadFile("test_responses/api_v2/feed_simple_tweet.json")
+	if err != nil {
+		panic(err)
+	}
+	var feed APIV2Response
 	err = json.Unmarshal(data, &feed)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -45,7 +81,7 @@ func TestAPIV2FeedSimpleTweet(t *testing.T) {
 	if tweet.ID != 1485708879174508550 {
 		t.Errorf("Expected ID 1485708879174508550, got %d", tweet.ID)
 	}
-	if tweet.UserID != scraper.UserID(44067298) {
+	if tweet.UserID != UserID(44067298) {
 		t.Errorf("Expected user ID 44067298, got %d", tweet.UserID)
 	}
 	expected_text := "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964"
@@ -61,11 +97,11 @@ func TestAPIV2FeedSimpleTweet(t *testing.T) {
 
 // Check a retweet
 func TestAPIV2FeedRetweet(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/feeds_api_v2/feed_simple_retweet.json")
+	data, err := ioutil.ReadFile("test_responses/api_v2/feed_simple_retweet.json")
 	if err != nil {
 		panic(err)
 	}
-	var feed scraper.APIV2Response
+	var feed APIV2Response
 	err = json.Unmarshal(data, &feed)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -107,7 +143,7 @@ func TestAPIV2FeedRetweet(t *testing.T) {
 	if tweet.ID != 1485694028620316673 {
 		t.Errorf("Expected ID %d, got %d", 1485694028620316673, tweet.ID)
 	}
-	if tweet.UserID != scraper.UserID(1326229737551912960) {
+	if tweet.UserID != UserID(1326229737551912960) {
 		t.Errorf("Expected user ID %d, got %d", 1326229737551912960, tweet.UserID)
 	}
 	expected_text := "More mask madness, this time in an elevator. The mask police are really nuts https://t.co/3BpvLjdJwD"
@@ -129,7 +165,7 @@ func TestAPIV2FeedRetweet(t *testing.T) {
 	if retweet.RetweetedAt.Unix() != 1643053397 {
 		t.Errorf("Expected retweeted_at %d, got %d", 1643053397, retweet.RetweetedAt.Unix())
 	}
-	if retweet.RetweetedByID != scraper.UserID(44067298) {
+	if retweet.RetweetedByID != UserID(44067298) {
 		t.Errorf("Expected retweeted_by 44067298, got %d", retweet.RetweetedByID)
 	}
 }
@@ -137,11 +173,11 @@ func TestAPIV2FeedRetweet(t *testing.T) {
 
 // Check a quote-tweet
 func TestAPIV2FeedQuoteTweet(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/feeds_api_v2/feed_quote_tweet.json")
+	data, err := ioutil.ReadFile("test_responses/api_v2/feed_quote_tweet.json")
 	if err != nil {
 		panic(err)
 	}
-	var feed scraper.APIV2Response
+	var feed APIV2Response
 	err = json.Unmarshal(data, &feed)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -200,11 +236,11 @@ func TestAPIV2FeedQuoteTweet(t *testing.T) {
 
 // Check a retweeted quote-tweet
 func TestAPIV2FeedRetweetedQuoteTweet(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/feeds_api_v2/feed_retweeted_quote_tweet.json")
+	data, err := ioutil.ReadFile("test_responses/api_v2/feed_retweeted_quote_tweet.json")
 	if err != nil {
 		panic(err)
 	}
-	var feed scraper.APIV2Response
+	var feed APIV2Response
 	err = json.Unmarshal(data, &feed)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -256,11 +292,11 @@ func TestAPIV2FeedRetweetedQuoteTweet(t *testing.T) {
 
 
 func TestParseAPIV2UserFeed(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/feeds_api_v2/user_feed_apiv2.json")
+	data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json")
 	if err != nil {
 		panic(err)
 	}
-	var feed scraper.APIV2Response
+	var feed APIV2Response
 	err = json.Unmarshal(data, &feed)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -304,6 +340,5 @@ func TestParseAPIV2UserFeed(t *testing.T) {
 		t.Errorf("Expected cursor %q, got %q", "HBaYgL2Fp/T7nCkAAA==", bottom_cursor)
 	}
 
-
-	println(len(tweet_trove.Users))
+	fmt.Printf("%d Users, %d Tweets, %d Retweets\n", len(tweet_trove.Users), len(tweet_trove.Tweets), len(tweet_trove.Retweets))
 }

scraper/test_responses/api_v2/user_michael_malice.json

@@ -0,0 +1 @@
+{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}}