From 54857f40cd58fd61c8dabbf18e6209a63f601696 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sat, 14 May 2022 15:02:15 -0700 Subject: [PATCH] Add parsing of tweets with Space links in them --- scraper/space.go | 8 +++----- scraper/space_test.go | 2 +- .../single_tweets/tweet_with_space_card.json | 1 + scraper/tweet.go | 11 +++++++++++ scraper/tweet_test.go | 10 ++++++++++ 5 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 scraper/test_responses/single_tweets/tweet_with_space_card.json diff --git a/scraper/space.go b/scraper/space.go index b41aab3..24aa7e0 100644 --- a/scraper/space.go +++ b/scraper/space.go @@ -3,16 +3,14 @@ package scraper type SpaceID string type Space struct { - ID SpaceID - TweetID TweetID - - Url string + ID SpaceID + ShortUrl string } func ParseAPISpace(apiCard APICard) Space { ret := Space{} ret.ID = SpaceID(apiCard.BindingValues.ID.StringValue) - ret.Url = apiCard.ShortenedUrl + ret.ShortUrl = apiCard.ShortenedUrl return ret } diff --git a/scraper/space_test.go b/scraper/space_test.go index b15208b..0b74f88 100644 --- a/scraper/space_test.go +++ b/scraper/space_test.go @@ -23,5 +23,5 @@ func TestParseSpace(t *testing.T) { space := ParseAPISpace(apiCard) assert.Equal(SpaceID("1YpKkZVyQjoxj"), space.ID) - assert.Equal("https://t.co/WBPAHNF8Om", space.Url) + assert.Equal("https://t.co/WBPAHNF8Om", space.ShortUrl) } diff --git a/scraper/test_responses/single_tweets/tweet_with_space_card.json b/scraper/test_responses/single_tweets/tweet_with_space_card.json new file mode 100644 index 0000000..e7175d6 --- /dev/null +++ b/scraper/test_responses/single_tweets/tweet_with_space_card.json @@ -0,0 +1 @@ +{"created_at":"Fri May 13 12:56:13 +0000 2022","id_str":"1525097585144238081","full_text":"🚨 SUNDAY! 🚨\n\nStrauss closed his 1932 notes on Schmitt by arguing that a critique of liberalism requires studying the horizon in which Hobbes founded it.\n\nIn 1935, he published his book on the genesis of Hobbes’ political philosophy.\n\nHow do they relate?\n\nhttps://t.co/WBPAHNF8Om","display_text_range":[0,278],"entities":{"urls":[{"url":"https://t.co/WBPAHNF8Om","expanded_url":"https://twitter.com/i/spaces/1YpKkZVyQjoxj","display_url":"twitter.com/i/spaces/1YpKk…","indices":[255,278]}]},"source":"Twitter for iPhone","user_id_str":"1681158044","retweet_count":6,"favorite_count":21,"reply_count":6,"quote_count":2,"conversation_id_str":"1525097585144238081","possibly_sensitive_editable":true,"card":{"name":"3691233323:audiospace","url":"https://t.co/WBPAHNF8Om","card_type_url":"http://card-type-url-is-deprecated.invalid","binding_values":{"id":{"type":"STRING","string_value":"1YpKkZVyQjoxj"},"narrow_cast_space_type":{"type":"STRING","string_value":"0"},"card_url":{"type":"STRING","string_value":"https://t.co/WBPAHNF8Om","scribe_key":"card_url"}},"card_platform":{"platform":{"device":{"name":"Swift","version":"12"},"audience":{"name":"production"}}}},"lang":"en","self_thread":{"id_str":"1525097585144238081"}} diff --git a/scraper/tweet.go b/scraper/tweet.go index 0f677ef..a4dfcff 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -33,6 +33,7 @@ type Tweet struct { Hashtags []string Urls []Url Polls []Poll + Spaces []Space TombstoneType string IsStub bool @@ -107,6 +108,10 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { for _, url := range apiTweet.Entities.URLs { var url_object Url if apiTweet.Card.ShortenedUrl == url.ShortenedUrl { + if apiTweet.Card.Name == "3691233323:audiospace" { + // This "url" is just a link to a Space. Don't process it as a Url + continue + } url_object = ParseAPIUrlCard(apiTweet.Card) } url_object.Text = url.ExpandedURL @@ -170,6 +175,12 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { ret.Polls = []Poll{poll} } + // Process spaces + if apiTweet.Card.Name == "3691233323:audiospace" { + space := ParseAPISpace(apiTweet.Card) + ret.Spaces = []Space{space} + } + // Process tombstones and other metadata ret.TombstoneType = apiTweet.TombstoneText ret.IsStub = !(ret.TombstoneType == "") diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 133ae53..a54a3eb 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -170,6 +170,16 @@ func TestTweetWithPoll(t *testing.T) { assert.Equal(int64(1638331935), p.LastUpdatedAt.Unix()) } +func TestTweetWithSpace(t *testing.T) { + assert := assert.New(t) + tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_space_card.json") + assert.Len(tweet.Urls, 0) + assert.Len(tweet.Spaces, 1) + + s := tweet.Spaces[0] + assert.Equal(SpaceID("1YpKkZVyQjoxj"), s.ID) +} + func TestParseTweetResponse(t *testing.T) { assert := assert.New(t) data, err := os.ReadFile("test_responses/michael_malice_feed.json")