Refactor API response types, creating new APIUser type

- extracted this from both UserResponse struct and TweetResponse struct,
both of which return something almost identical

scraper/api_types.go

@@ -85,10 +85,8 @@ func (t APITweet) String() string {
 	return string(data)
 }
 
-type TweetResponse struct {
+
-	GlobalObjects struct {
+type APIUser struct {
-		Tweets map[string]APITweet `json:"tweets"`
-		Users  map[string]struct {
 	CreatedAt   string `json:"created_at"`
 	Description string `json:"description"`
 	Entities    struct {
@@ -112,7 +110,27 @@ type TweetResponse struct {
 	ScreenName           string   `json:"screen_name"`
 	StatusesCount        int      `json:"statuses_count"`
 	Verified             bool     `json:"verified"`
-		} `json:"users"`
+}
+
+
+type UserResponse struct {
+	Data struct {
+		User struct {
+			ID     string  `json:"rest_id"`
+			Legacy APIUser `json:"legacy"`
+		} `json:"user"`
+	} `json:"data"`
+}
+func (u UserResponse) ConvertToAPIUser() APIUser {
+	ret := u.Data.User.Legacy
+	ret.IDStr = u.Data.User.ID
+	return ret
+}
+
+type TweetResponse struct {
+	GlobalObjects struct {
+		Tweets map[string]APITweet `json:"tweets"`
+		Users  map[string]APIUser  `json:"users"`
 	} `json:"globalObjects"`
 	Timeline struct {
 		Instructions []struct {
@@ -140,36 +158,3 @@ func (t *TweetResponse) GetCursor() string {
 	}
 	return ""
 }
-
-
-type UserResponse struct {
-	Data struct {
-		User struct {
-			ID     string `json:"rest_id"`
-			Legacy struct {
-				CreatedAt   string `json:"created_at"`
-				Description string `json:"description"`
-				Entities    struct {
-					URL struct {
-						Urls []struct {
-							ExpandedURL string `json:"expanded_url"`
-						} `json:"urls"`
-					} `json:"url"`
-				} `json:"entities"`
-				FavouritesCount      int      `json:"favourites_count"`
-				FollowersCount       int      `json:"followers_count"`
-				FriendsCount         int      `json:"friends_count"`
-				ListedCount          int      `json:"listed_count"`
-				Name                 string   `json:"name"`
-				Location             string   `json:"location"`
-				PinnedTweetIdsStr    []string `json:"pinned_tweet_ids_str"`
-				ProfileBannerURL     string   `json:"profile_banner_url"`
-				ProfileImageURLHTTPS string   `json:"profile_image_url_https"`
-				Protected            bool     `json:"protected"`
-				ScreenName           string   `json:"screen_name"`
-				StatusesCount        int      `json:"statuses_count"`
-				Verified             bool     `json:"verified"`
-			} `json:"legacy"`
-		} `json:"user"`
-	} `json:"data"`
-}

scraper/api_types_test.go

@@ -38,6 +38,28 @@ func TestNormalizeContent(t *testing.T) {
 }
 
 
+func TestUserProfileToAPIUser(t *testing.T) {
+	data, err := ioutil.ReadFile("test_responses/michael_malice_user_profile.json")
+	if err != nil {
+		panic(err)
+	}
+	var user_resp scraper.UserResponse
+	err = json.Unmarshal(data, &user_resp)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	result := user_resp.ConvertToAPIUser()
+
+	if result.IDStr != "44067298" {
+		t.Errorf("Expected IDStr %q, got %q", "44067298", result.IDStr)
+	}
+	if result.FollowersCount != user_resp.Data.User.Legacy.FollowersCount {
+		t.Errorf("Expected user count %d, got %d", user_resp.Data.User.Legacy.FollowersCount, result.FollowersCount)
+	}
+}
+
+
 func TestGetCursor(t *testing.T) {
 	data, err := ioutil.ReadFile("test_responses/midriffs_anarchist_cookbook.json")
 	if err != nil {