Add parsing of tweets with Space links in them

scraper/space.go

@@ -3,16 +3,14 @@ package scraper
 type SpaceID string
 
 type Space struct {
-	ID      SpaceID
-	TweetID TweetID
-
-	Url string
+	ID       SpaceID
+	ShortUrl string
 }
 
 func ParseAPISpace(apiCard APICard) Space {
 	ret := Space{}
 	ret.ID = SpaceID(apiCard.BindingValues.ID.StringValue)
-	ret.Url = apiCard.ShortenedUrl
+	ret.ShortUrl = apiCard.ShortenedUrl
 
 	return ret
 }

scraper/space_test.go

@@ -23,5 +23,5 @@ func TestParseSpace(t *testing.T) {
 
 	space := ParseAPISpace(apiCard)
 	assert.Equal(SpaceID("1YpKkZVyQjoxj"), space.ID)
-	assert.Equal("https://t.co/WBPAHNF8Om", space.Url)
+	assert.Equal("https://t.co/WBPAHNF8Om", space.ShortUrl)
 }

scraper/test_responses/single_tweets/tweet_with_space_card.json

@@ -0,0 +1 @@
+{"created_at":"Fri May 13 12:56:13 +0000 2022","id_str":"1525097585144238081","full_text":"🚨 SUNDAY! 🚨\n\nStrauss closed his 1932 notes on Schmitt by arguing that a critique of liberalism requires studying the horizon in which Hobbes founded it.\n\nIn 1935, he published his book on the genesis of Hobbes’ political philosophy.\n\nHow do they relate?\n\nhttps://t.co/WBPAHNF8Om","display_text_range":[0,278],"entities":{"urls":[{"url":"https://t.co/WBPAHNF8Om","expanded_url":"https://twitter.com/i/spaces/1YpKkZVyQjoxj","display_url":"twitter.com/i/spaces/1YpKk…","indices":[255,278]}]},"source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","user_id_str":"1681158044","retweet_count":6,"favorite_count":21,"reply_count":6,"quote_count":2,"conversation_id_str":"1525097585144238081","possibly_sensitive_editable":true,"card":{"name":"3691233323:audiospace","url":"https://t.co/WBPAHNF8Om","card_type_url":"http://card-type-url-is-deprecated.invalid","binding_values":{"id":{"type":"STRING","string_value":"1YpKkZVyQjoxj"},"narrow_cast_space_type":{"type":"STRING","string_value":"0"},"card_url":{"type":"STRING","string_value":"https://t.co/WBPAHNF8Om","scribe_key":"card_url"}},"card_platform":{"platform":{"device":{"name":"Swift","version":"12"},"audience":{"name":"production"}}}},"lang":"en","self_thread":{"id_str":"1525097585144238081"}}

scraper/tweet.go

@@ -33,6 +33,7 @@ type Tweet struct {
 	Hashtags      []string
 	Urls          []Url
 	Polls         []Poll
+	Spaces        []Space
 
 	TombstoneType string
 	IsStub        bool
@@ -107,6 +108,10 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
 	for _, url := range apiTweet.Entities.URLs {
 		var url_object Url
 		if apiTweet.Card.ShortenedUrl == url.ShortenedUrl {
+			if apiTweet.Card.Name == "3691233323:audiospace" {
+				// This "url" is just a link to a Space.  Don't process it as a Url
+				continue
+			}
 			url_object = ParseAPIUrlCard(apiTweet.Card)
 		}
 		url_object.Text = url.ExpandedURL
@@ -170,6 +175,12 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
 		ret.Polls = []Poll{poll}
 	}
 
+	// Process spaces
+	if apiTweet.Card.Name == "3691233323:audiospace" {
+		space := ParseAPISpace(apiTweet.Card)
+		ret.Spaces = []Space{space}
+	}
+
 	// Process tombstones and other metadata
 	ret.TombstoneType = apiTweet.TombstoneText
 	ret.IsStub = !(ret.TombstoneType == "")

scraper/tweet_test.go

@@ -170,6 +170,16 @@ func TestTweetWithPoll(t *testing.T) {
 	assert.Equal(int64(1638331935), p.LastUpdatedAt.Unix())
 }
 
+func TestTweetWithSpace(t *testing.T) {
+	assert := assert.New(t)
+	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_space_card.json")
+	assert.Len(tweet.Urls, 0)
+	assert.Len(tweet.Spaces, 1)
+
+	s := tweet.Spaces[0]
+	assert.Equal(SpaceID("1YpKkZVyQjoxj"), s.ID)
+}
+
 func TestParseTweetResponse(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/michael_malice_feed.json")