Add parsing of Space object

doc/curl requests

@@ -44,3 +44,12 @@ curl \
     -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
     -H "X-Guest-Token: 1449946080792104970" \
     "https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q=potatoes"
+
+
+
+#
+# A twitter Space:
+curl \
+    -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
+    -H "X-Guest-Token: 1591520847784706050" \
+    "https://twitter.com/i/api/graphql/Ha9BKBF0uAz9d4-lz0jnYA/AudioSpaceById?variables=%7B%22id%22%3A%221BdxYypQzBgxX%22%2C%22isMetatagsQuery%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withReplays%22%3Atrue%7D&features=%7B%22spaces_2022_h2_clipping%22%3Atrue%2C%22spaces_2022_h2_spaces_communities%22%3Atrue%2C%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D"

scraper/api_types_v2.go

@@ -485,3 +485,85 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo
 	}
 	return nil
 }
+
+type SpaceResponse struct {
+	Data struct {
+		AudioSpace struct {
+			Metadata struct {
+				RestId                      string `json:"rest_id"`
+				State                       string
+				Title                       string
+				MediaKey                    string `json:"media_key"`
+				CreatedAt                   int64  `json:"created_at"`
+				StartedAt                   int64  `json:"started_at"`
+				EndedAt                     int64  `json:"ended_at,string"`
+				UpdatedAt                   int64  `json:"updated_at"`
+				DisallowJoin                bool   `json:"disallow_join"`
+				NarrowCastSpaceType         int64  `json:"narrow_cast_space_type"`
+				IsEmployeeOnly              bool   `json:"is_employee_only"`
+				IsLocked                    bool   `json:"is_locked"`
+				IsSpaceAvailableForReplay   bool   `json:"is_space_available_for_replay"`
+				IsSpaceAvailableForClipping bool   `json:"is_space_available_for_clipping"`
+				ConversationControls        int64  `json:"conversation_controls"`
+				TotalReplayWatched          int64  `json:"total_replay_watched"`
+				TotalLiveListeners          int64  `json:"total_live_listeners"`
+				CreatorResults              struct {
+					Result struct {
+						ID     int64   `json:"rest_id,string"`
+						Legacy APIUser `json:"legacy"`
+					} `json:"result"`
+				} `json:"creator_results"`
+			}
+			Participants struct {
+				Total  int
+				Admins []struct {
+					Start int
+					User  struct {
+						RestId int64 `json:"rest_id,string"`
+					}
+				}
+				Speakers []struct {
+					User struct {
+						RestId int64 `json:"rest_id,string"`
+					}
+				}
+			}
+		}
+	}
+}
+
+func (r SpaceResponse) ToTweetTrove() TweetTrove {
+	data := r.Data.AudioSpace
+
+	ret := NewTweetTrove()
+	space := Space{}
+	space.ID = SpaceID(data.Metadata.RestId)
+	space.Title = data.Metadata.Title
+	space.State = data.Metadata.State
+	space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt)
+	space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt)
+	space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt)
+	space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt)
+	space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay
+	space.ReplayWatchCount = data.Metadata.TotalReplayWatched
+	space.LiveListenersCount = data.Metadata.TotalLiveListeners
+	space.IsDetailsFetched = true
+
+	for _, admin := range data.Participants.Admins {
+		space.ParticipantIds = append(space.ParticipantIds, UserID(admin.User.RestId))
+	}
+	for _, speaker := range data.Participants.Speakers {
+		space.ParticipantIds = append(space.ParticipantIds, UserID(speaker.User.RestId))
+	}
+
+	ret.Spaces[space.ID] = space
+
+	creator, err := ParseSingleUser(data.Metadata.CreatorResults.Result.Legacy)
+	if err != nil {
+		panic(err)
+	}
+	creator.ID = UserID(data.Metadata.CreatorResults.Result.ID)
+	ret.Users[creator.ID] = creator
+
+	return ret
+}

scraper/api_types_v2_test.go

@@ -463,6 +463,42 @@ func TestAPIV2ParseTweetWithSpace(t *testing.T) {
 
 	s := tweet.Spaces[0]
 	assert.Equal(SpaceID("1dRJZlRNZDzKB"), s.ID)
+	assert.Equal("https://t.co/5RLbEwQgvH", s.ShortUrl)
+	assert.False(s.IsDetailsFetched)
+}
+
+func TestParseSpaceResponse(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+	data, err := os.ReadFile("test_responses/tweet_content/space_object.json")
+	if err != nil {
+		panic(err)
+	}
+
+	var response SpaceResponse
+	err = json.Unmarshal(data, &response)
+	assert.NoError(err)
+
+	trove := response.ToTweetTrove()
+	require.Len(trove.Spaces, 1)
+	space := trove.Spaces["1BdxYypQzBgxX"]
+	assert.Equal(space.Title, "dreary weather 🌧️☔🌬️")
+	assert.Equal(int64(1665884387263), space.CreatedAt.Time.Unix())
+	assert.Equal(int64(1665884388222), space.StartedAt.Time.Unix())
+	assert.Equal(int64(1665887491804), space.EndedAt.Time.Unix())
+	assert.Equal(int64(1665887492705), space.UpdatedAt.Time.Unix())
+	assert.False(space.IsAvailableForReplay)
+	assert.Equal(int64(4), space.ReplayWatchCount)
+	assert.Equal(int64(1), space.LiveListenersCount)
+	assert.True(space.IsDetailsFetched)
+
+	assert.Len(space.ParticipantIds, 2)
+	assert.Equal(UserID(1356335022815539201), space.ParticipantIds[0])
+	assert.Equal(UserID(1523838615377350656), space.ParticipantIds[1])
+
+	require.Len(trove.Users, 1)
+	user := trove.Users[1356335022815539201]
+	assert.Equal(847, user.FollowersCount)
 }
 
 func TestParseAPIV2UserFeed(t *testing.T) {

scraper/space.go

@@ -5,6 +5,21 @@ type SpaceID string
 type Space struct {
 	ID                   SpaceID   `db:"id"`
 	ShortUrl             string    `db:"short_url"`
+	State                string    `db:"state"`
+	Title                string    `db:"title"`
+	CreatedAt            Timestamp `db:"created_at"`
+	StartedAt            Timestamp
+	EndedAt              Timestamp `db:"ended_at"`
+	UpdatedAt            Timestamp
+	IsAvailableForReplay bool
+	ReplayWatchCount     int64
+	LiveListenersCount   int64
+	ParticipantIds       []UserID
+
+	CreatedById UserID
+	TweetID     TweetID
+
+	IsDetailsFetched bool
 }
 
 func ParseAPISpace(apiCard APICard) Space {
@@ -12,5 +27,8 @@ func ParseAPISpace(apiCard APICard) Space {
 	ret.ID = SpaceID(apiCard.BindingValues.ID.StringValue)
 	ret.ShortUrl = apiCard.ShortenedUrl
 
+	// Indicate that this Space needs its details fetched still
+	ret.IsDetailsFetched = false
+
 	return ret
 }

scraper/test_responses/tweet_content/space_object.json

@@ -0,0 +1 @@
+{"data":{"audioSpace":{"metadata":{"rest_id":"1BdxYypQzBgxX","state":"Ended","title":"dreary weather 🌧️☔🌬️","media_key":"28_1581459859551449088","created_at":1665884387263,"started_at":1665884388222,"ended_at":"1665887491804","updated_at":1665887492705,"disallow_join":false,"narrow_cast_space_type":0,"is_employee_only":false,"is_locked":false,"is_space_available_for_replay":false,"is_space_available_for_clipping":false,"conversation_controls":0,"total_replay_watched":4,"total_live_listeners":1,"creator_results":{"result":{"__typename":"User","id":"VXNlcjoxMzU2MzM1MDIyODE1NTM5MjAx","rest_id":"1356335022815539201","affiliates_highlighted_label":{},"has_nft_avatar":false,"is_blue_verified":false,"legacy":{"created_at":"Mon Feb 01 20:14:13 +0000 2021","default_profile":true,"default_profile_image":false,"description":"Atomized millennial • Class reductionist • Homosexual male • Abolish plastic","entities":{"description":{"urls":[]}},"fast_followers_count":0,"favourites_count":67897,"followers_count":847,"friends_count":677,"has_custom_timelines":false,"is_translator":false,"listed_count":3,"location":"","media_count":1524,"name":"Roy","normal_followers_count":847,"pinned_tweet_ids_str":["1427396378410594306"],"possibly_sensitive":false,"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":61.77,"rgb":{"blue":134,"green":134,"red":139}},{"percentage":36.05,"rgb":{"blue":39,"green":35,"red":35}},{"percentage":0.87,"rgb":{"blue":133,"green":163,"red":196}},{"percentage":0.85,"rgb":{"blue":124,"green":112,"red":109}},{"percentage":0.45,"rgb":{"blue":194,"green":216,"red":230}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/1356335022815539201/1639441025","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":96.92,"rgb":{"blue":152,"green":180,"red":187}},{"percentage":2.66,"rgb":{"blue":63,"green":92,"red":113}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1549170001113907201/g5HNCvxM_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"royllovians","statuses_count":14486,"translator_type":"none","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"sharings":{"items":[],"slice_info":{}},"participants":{"total":0,"admins":[{"periscope_user_id":"1raQZLXDVBpjz","start":1665884387263,"twitter_screen_name":"royllovians","display_name":"Roy","avatar_url":"https://pbs.twimg.com/profile_images/1549170001113907201/g5HNCvxM_normal.jpg","is_verified":false,"is_muted_by_admin":false,"is_muted_by_guest":true,"user_results":{"result":{"__typename":"User","has_nft_avatar":false,"is_blue_verified":false}},"user":{"rest_id":"1356335022815539201"}}],"speakers":[{"periscope_user_id":"1PXEdxenBZlEe","start":1665886564026,"twitter_screen_name":"BlkPHomo","display_name":"Bphm","avatar_url":"https://pbs.twimg.com/profile_images/1523839095428034560/I60Ihsoa_normal.jpg","is_verified":false,"is_muted_by_admin":false,"is_muted_by_guest":false,"user_results":{"result":{"__typename":"User","has_nft_avatar":false,"is_blue_verified":false}},"user":{"rest_id":"1523838615377350656"}}],"listeners":[]}}}}

scraper/tweet_test.go

@@ -213,6 +213,8 @@ func TestTweetWithSpace(t *testing.T) {
 
 	s := tweet.Spaces[0]
 	assert.Equal(SpaceID("1YpKkZVyQjoxj"), s.ID)
+	assert.Equal("https://t.co/WBPAHNF8Om", s.ShortUrl)
+	assert.False(s.IsDetailsFetched)
 }
 
 func TestParseTweetResponse(t *testing.T) {

scraper/tweet_trove.go

@@ -11,6 +11,7 @@ type TweetTrove struct {
 	Tweets   map[TweetID]Tweet
 	Users    map[UserID]User
 	Retweets map[TweetID]Retweet
+	Spaces   map[SpaceID]Space
 
 	TombstoneUsers []UserHandle
 }
@@ -20,6 +21,7 @@ func NewTweetTrove() TweetTrove {
 	ret.Tweets = make(map[TweetID]Tweet)
 	ret.Users = make(map[UserID]User)
 	ret.Retweets = make(map[TweetID]Retweet)
+	ret.Spaces = make(map[SpaceID]Space)
 	ret.TombstoneUsers = []UserHandle{}
 	return ret
 }
@@ -49,6 +51,9 @@ func (t1 *TweetTrove) MergeWith(t2 TweetTrove) {
 	for id, val := range t2.Retweets {
 		t1.Retweets[id] = val
 	}
+	for id, val := range t2.Spaces {
+		t1.Spaces[id] = val
+	}
 
 	t1.TombstoneUsers = append(t1.TombstoneUsers, t2.TombstoneUsers...)
 }