From 2a45818468764ee974d8ffd45c4a4b79af334028 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 13 Nov 2022 13:17:47 -0500 Subject: [PATCH] Add parsing of Space object --- doc/curl requests | 9 ++ scraper/api_types_v2.go | 82 +++++++++++++++++++ scraper/api_types_v2_test.go | 36 ++++++++ scraper/space.go | 22 ++++- .../tweet_content/space_object.json | 1 + scraper/tweet_test.go | 2 + scraper/tweet_trove.go | 5 ++ 7 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 scraper/test_responses/tweet_content/space_object.json diff --git a/doc/curl requests b/doc/curl requests index b431a2c..12cc29e 100644 --- a/doc/curl requests +++ b/doc/curl requests @@ -44,3 +44,12 @@ curl \ -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ -H "X-Guest-Token: 1449946080792104970" \ "https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q=potatoes" + + + +# +# A twitter Space: +curl \ + -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ + -H "X-Guest-Token: 1591520847784706050" \ + "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%221BdxYypQzBgxX%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D" diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 7c081d4..2bb3ff4 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -485,3 +485,85 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo } return nil } + +type SpaceResponse struct { + Data struct { + AudioSpace struct { + Metadata struct { + RestId string `json:"rest_id"` + State string + Title string + MediaKey string `json:"media_key"` + CreatedAt int64 `json:"created_at"` + StartedAt int64 `json:"started_at"` + EndedAt int64 `json:"ended_at,string"` + UpdatedAt int64 `json:"updated_at"` + DisallowJoin bool `json:"disallow_join"` + NarrowCastSpaceType int64 `json:"narrow_cast_space_type"` + IsEmployeeOnly bool `json:"is_employee_only"` + IsLocked bool `json:"is_locked"` + IsSpaceAvailableForReplay bool `json:"is_space_available_for_replay"` + IsSpaceAvailableForClipping bool `json:"is_space_available_for_clipping"` + ConversationControls int64 `json:"conversation_controls"` + TotalReplayWatched int64 `json:"total_replay_watched"` + TotalLiveListeners int64 `json:"total_live_listeners"` + CreatorResults struct { + Result struct { + ID int64 `json:"rest_id,string"` + Legacy APIUser `json:"legacy"` + } `json:"result"` + } `json:"creator_results"` + } + Participants struct { + Total int + Admins []struct { + Start int + User struct { + RestId int64 `json:"rest_id,string"` + } + } + Speakers []struct { + User struct { + RestId int64 `json:"rest_id,string"` + } + } + } + } + } +} + +func (r SpaceResponse) ToTweetTrove() TweetTrove { + data := r.Data.AudioSpace + + ret := NewTweetTrove() + space := Space{} + space.ID = SpaceID(data.Metadata.RestId) + space.Title = data.Metadata.Title + space.State = data.Metadata.State + space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt) + space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt) + space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt) + space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt) + space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay + space.ReplayWatchCount = data.Metadata.TotalReplayWatched + space.LiveListenersCount = data.Metadata.TotalLiveListeners + space.IsDetailsFetched = true + + for _, admin := range data.Participants.Admins { + space.ParticipantIds = append(space.ParticipantIds, UserID(admin.User.RestId)) + } + for _, speaker := range data.Participants.Speakers { + space.ParticipantIds = append(space.ParticipantIds, UserID(speaker.User.RestId)) + } + + ret.Spaces[space.ID] = space + + creator, err := ParseSingleUser(data.Metadata.CreatorResults.Result.Legacy) + if err != nil { + panic(err) + } + creator.ID = UserID(data.Metadata.CreatorResults.Result.ID) + ret.Users[creator.ID] = creator + + return ret +} diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index 78790bc..75b9f46 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -463,6 +463,42 @@ func TestAPIV2ParseTweetWithSpace(t *testing.T) { s := tweet.Spaces[0] assert.Equal(SpaceID("1dRJZlRNZDzKB"), s.ID) + assert.Equal("https://t.co/5RLbEwQgvH", s.ShortUrl) + assert.False(s.IsDetailsFetched) +} + +func TestParseSpaceResponse(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + data, err := os.ReadFile("test_responses/tweet_content/space_object.json") + if err != nil { + panic(err) + } + + var response SpaceResponse + err = json.Unmarshal(data, &response) + assert.NoError(err) + + trove := response.ToTweetTrove() + require.Len(trove.Spaces, 1) + space := trove.Spaces["1BdxYypQzBgxX"] + assert.Equal(space.Title, "dreary weather 🌧️☔🌬️") + assert.Equal(int64(1665884387263), space.CreatedAt.Time.Unix()) + assert.Equal(int64(1665884388222), space.StartedAt.Time.Unix()) + assert.Equal(int64(1665887491804), space.EndedAt.Time.Unix()) + assert.Equal(int64(1665887492705), space.UpdatedAt.Time.Unix()) + assert.False(space.IsAvailableForReplay) + assert.Equal(int64(4), space.ReplayWatchCount) + assert.Equal(int64(1), space.LiveListenersCount) + assert.True(space.IsDetailsFetched) + + assert.Len(space.ParticipantIds, 2) + assert.Equal(UserID(1356335022815539201), space.ParticipantIds[0]) + assert.Equal(UserID(1523838615377350656), space.ParticipantIds[1]) + + require.Len(trove.Users, 1) + user := trove.Users[1356335022815539201] + assert.Equal(847, user.FollowersCount) } func TestParseAPIV2UserFeed(t *testing.T) { diff --git a/scraper/space.go b/scraper/space.go index 70b5394..7a0a2f9 100644 --- a/scraper/space.go +++ b/scraper/space.go @@ -3,8 +3,23 @@ package scraper type SpaceID string type Space struct { - ID SpaceID `db:"id"` - ShortUrl string `db:"short_url"` + ID SpaceID `db:"id"` + ShortUrl string `db:"short_url"` + State string `db:"state"` + Title string `db:"title"` + CreatedAt Timestamp `db:"created_at"` + StartedAt Timestamp + EndedAt Timestamp `db:"ended_at"` + UpdatedAt Timestamp + IsAvailableForReplay bool + ReplayWatchCount int64 + LiveListenersCount int64 + ParticipantIds []UserID + + CreatedById UserID + TweetID TweetID + + IsDetailsFetched bool } func ParseAPISpace(apiCard APICard) Space { @@ -12,5 +27,8 @@ func ParseAPISpace(apiCard APICard) Space { ret.ID = SpaceID(apiCard.BindingValues.ID.StringValue) ret.ShortUrl = apiCard.ShortenedUrl + // Indicate that this Space needs its details fetched still + ret.IsDetailsFetched = false + return ret } diff --git a/scraper/test_responses/tweet_content/space_object.json b/scraper/test_responses/tweet_content/space_object.json new file mode 100644 index 0000000..62e2f2d --- /dev/null +++ b/scraper/test_responses/tweet_content/space_object.json @@ -0,0 +1 @@ +{"data":{"audioSpace":{"metadata":{"rest_id":"1BdxYypQzBgxX","state":"Ended","title":"dreary weather 🌧️☔🌬️","media_key":"28_1581459859551449088","created_at":1665884387263,"started_at":1665884388222,"ended_at":"1665887491804","updated_at":1665887492705,"disallow_join":false,"narrow_cast_space_type":0,"is_employee_only":false,"is_locked":false,"is_space_available_for_replay":false,"is_space_available_for_clipping":false,"conversation_controls":0,"total_replay_watched":4,"total_live_listeners":1,"creator_results":{"result":{"__typename":"User","id":"VXNlcjoxMzU2MzM1MDIyODE1NTM5MjAx","rest_id":"1356335022815539201","affiliates_highlighted_label":{},"has_nft_avatar":false,"is_blue_verified":false,"legacy":{"created_at":"Mon Feb 01 20:14:13 +0000 2021","default_profile":true,"default_profile_image":false,"description":"Atomized millennial • Class reductionist • Homosexual male • Abolish plastic","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":67897,"followers_count":847,"friends_count":677,"has_custom_timelines":false,"is_translator":false,"listed_count":3,"location":"","media_count":1524,"name":"Roy","normal_followers_count":847,"pinned_tweet_ids_str":["1427396378410594306"],"possibly_sensitive":false,"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":61.77,"rgb":{"blue":134,"green":134,"red":139}},{"percentage":36.05,"rgb":{"blue":39,"green":35,"red":35}},{"percentage":0.87,"rgb":{"blue":133,"green":163,"red":196}},{"percentage":0.85,"rgb":{"blue":124,"green":112,"red":109}},{"percentage":0.45,"rgb":{"blue":194,"green":216,"red":230}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1356335022815539201/1639441025","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":96.92,"rgb":{"blue":152,"green":180,"red":187}},{"percentage":2.66,"rgb":{"blue":63,"green":92,"red":113}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1549170001113907201/g5HNCvxM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"royllovians","statuses_count":14486,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"sharings":{"items":[],"slice_info":{}},"participants":{"total":0,"admins":[{"periscope_user_id":"1raQZLXDVBpjz","start":1665884387263,"twitter_screen_name":"royllovians","display_name":"Roy","avatar_url":"https://pbs.twimg.com/profile_images/1549170001113907201/g5HNCvxM_normal.jpg","is_verified":false,"is_muted_by_admin":false,"is_muted_by_guest":true,"user_results":{"result":{"__typename":"User","has_nft_avatar":false,"is_blue_verified":false}},"user":{"rest_id":"1356335022815539201"}}],"speakers":[{"periscope_user_id":"1PXEdxenBZlEe","start":1665886564026,"twitter_screen_name":"BlkPHomo","display_name":"Bphm","avatar_url":"https://pbs.twimg.com/profile_images/1523839095428034560/I60Ihsoa_normal.jpg","is_verified":false,"is_muted_by_admin":false,"is_muted_by_guest":false,"user_results":{"result":{"__typename":"User","has_nft_avatar":false,"is_blue_verified":false}},"user":{"rest_id":"1523838615377350656"}}],"listeners":[]}}}} diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 87e2e2a..cd27600 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -213,6 +213,8 @@ func TestTweetWithSpace(t *testing.T) { s := tweet.Spaces[0] assert.Equal(SpaceID("1YpKkZVyQjoxj"), s.ID) + assert.Equal("https://t.co/WBPAHNF8Om", s.ShortUrl) + assert.False(s.IsDetailsFetched) } func TestParseTweetResponse(t *testing.T) { diff --git a/scraper/tweet_trove.go b/scraper/tweet_trove.go index 6c11e51..57e98d4 100644 --- a/scraper/tweet_trove.go +++ b/scraper/tweet_trove.go @@ -11,6 +11,7 @@ type TweetTrove struct { Tweets map[TweetID]Tweet Users map[UserID]User Retweets map[TweetID]Retweet + Spaces map[SpaceID]Space TombstoneUsers []UserHandle } @@ -20,6 +21,7 @@ func NewTweetTrove() TweetTrove { ret.Tweets = make(map[TweetID]Tweet) ret.Users = make(map[UserID]User) ret.Retweets = make(map[TweetID]Retweet) + ret.Spaces = make(map[SpaceID]Space) ret.TombstoneUsers = []UserHandle{} return ret } @@ -49,6 +51,9 @@ func (t1 *TweetTrove) MergeWith(t2 TweetTrove) { for id, val := range t2.Retweets { t1.Retweets[id] = val } + for id, val := range t2.Spaces { + t1.Spaces[id] = val + } t1.TombstoneUsers = append(t1.TombstoneUsers, t2.TombstoneUsers...) }