diff --git a/scraper/api_types.go b/scraper/api_types.go index 28a5c4a..7072093 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -85,34 +85,52 @@ func (t APITweet) String() string { return string(data) } + +type APIUser struct { + CreatedAt string `json:"created_at"` + Description string `json:"description"` + Entities struct { + URL struct { + Urls []struct { + ExpandedURL string `json:"expanded_url"` + } `json:"urls"` + } `json:"url"` + } `json:"entities"` + FavouritesCount int `json:"favourites_count"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + IDStr string `json:"id_str"` + ListedCount int `json:"listed_count"` + Name string `json:"name"` + Location string `json:"location"` + PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + Protected bool `json:"protected"` + ScreenName string `json:"screen_name"` + StatusesCount int `json:"statuses_count"` + Verified bool `json:"verified"` +} + + +type UserResponse struct { + Data struct { + User struct { + ID string `json:"rest_id"` + Legacy APIUser `json:"legacy"` + } `json:"user"` + } `json:"data"` +} +func (u UserResponse) ConvertToAPIUser() APIUser { + ret := u.Data.User.Legacy + ret.IDStr = u.Data.User.ID + return ret +} + type TweetResponse struct { GlobalObjects struct { Tweets map[string]APITweet `json:"tweets"` - Users map[string]struct { - CreatedAt string `json:"created_at"` - Description string `json:"description"` - Entities struct { - URL struct { - Urls []struct { - ExpandedURL string `json:"expanded_url"` - } `json:"urls"` - } `json:"url"` - } `json:"entities"` - FavouritesCount int `json:"favourites_count"` - FollowersCount int `json:"followers_count"` - FriendsCount int `json:"friends_count"` - IDStr string `json:"id_str"` - ListedCount int `json:"listed_count"` - Name string `json:"name"` - Location string `json:"location"` - PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` - ProfileBannerURL string `json:"profile_banner_url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - Protected bool `json:"protected"` - ScreenName string `json:"screen_name"` - StatusesCount int `json:"statuses_count"` - Verified bool `json:"verified"` - } `json:"users"` + Users map[string]APIUser `json:"users"` } `json:"globalObjects"` Timeline struct { Instructions []struct { @@ -140,36 +158,3 @@ func (t *TweetResponse) GetCursor() string { } return "" } - - -type UserResponse struct { - Data struct { - User struct { - ID string `json:"rest_id"` - Legacy struct { - CreatedAt string `json:"created_at"` - Description string `json:"description"` - Entities struct { - URL struct { - Urls []struct { - ExpandedURL string `json:"expanded_url"` - } `json:"urls"` - } `json:"url"` - } `json:"entities"` - FavouritesCount int `json:"favourites_count"` - FollowersCount int `json:"followers_count"` - FriendsCount int `json:"friends_count"` - ListedCount int `json:"listed_count"` - Name string `json:"name"` - Location string `json:"location"` - PinnedTweetIdsStr []string `json:"pinned_tweet_ids_str"` - ProfileBannerURL string `json:"profile_banner_url"` - ProfileImageURLHTTPS string `json:"profile_image_url_https"` - Protected bool `json:"protected"` - ScreenName string `json:"screen_name"` - StatusesCount int `json:"statuses_count"` - Verified bool `json:"verified"` - } `json:"legacy"` - } `json:"user"` - } `json:"data"` -} diff --git a/scraper/api_types_test.go b/scraper/api_types_test.go index c3bdbd2..ba87439 100644 --- a/scraper/api_types_test.go +++ b/scraper/api_types_test.go @@ -38,6 +38,28 @@ func TestNormalizeContent(t *testing.T) { } +func TestUserProfileToAPIUser(t *testing.T) { + data, err := ioutil.ReadFile("test_responses/michael_malice_user_profile.json") + if err != nil { + panic(err) + } + var user_resp scraper.UserResponse + err = json.Unmarshal(data, &user_resp) + if err != nil { + t.Errorf(err.Error()) + } + + result := user_resp.ConvertToAPIUser() + + if result.IDStr != "44067298" { + t.Errorf("Expected IDStr %q, got %q", "44067298", result.IDStr) + } + if result.FollowersCount != user_resp.Data.User.Legacy.FollowersCount { + t.Errorf("Expected user count %d, got %d", user_resp.Data.User.Legacy.FollowersCount, result.FollowersCount) + } +} + + func TestGetCursor(t *testing.T) { data, err := ioutil.ReadFile("test_responses/midriffs_anarchist_cookbook.json") if err != nil {