From e932dd33d5f0a38c7aad3bc78f720aea55a8776c Mon Sep 17 00:00:00 2001 From: Alessio Date: Sat, 14 May 2022 17:36:47 -0700 Subject: [PATCH] Add parsing Spaces for API v2 --- scraper/api_types_v2.go | 24 +++++++++++++++++ scraper/api_types_v2_test.go | 26 +++++++++++++++++++ .../api_v2/tweet_with_audiospaces_card.json | 1 + 3 files changed, 51 insertions(+) create mode 100644 scraper/test_responses/api_v2/tweet_with_audiospaces_card.json diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 7880a62..6f87ded 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -113,6 +113,17 @@ func (card APIV2Card) ParseAsPoll() Poll { } return ret } +func (card APIV2Card) ParseAsSpace() Space { + values := make(map[string]CardValue) + for _, obj := range card.Legacy.BindingValues { + values[obj.Key] = obj.Value + } + ret := Space{} + ret.ID = SpaceID(values["id"].StringValue) + ret.ShortUrl = values["card_url"].StringValue + + return ret +} type APIV2UserResult struct { UserResults struct { @@ -235,6 +246,19 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove poll := api_result.Result.Card.ParseAsPoll() poll.TweetID = main_tweet.ID main_tweet.Polls = []Poll{poll} + ret.Tweets[main_tweet.ID] = main_tweet + } else if api_result.Result.Card.Legacy.Name == "3691233323:audiospace" { + space := api_result.Result.Card.ParseAsSpace() + main_tweet.Spaces = []Space{space} + + // Remove it from the Urls + for i, url := range main_tweet.Urls { + if url.ShortText == space.ShortUrl { + main_tweet.Urls = append(main_tweet.Urls[:i], main_tweet.Urls[i+1:]...) + break + } + } + ret.Tweets[main_tweet.ID] = main_tweet } } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index f498351..78790bc 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -439,6 +439,32 @@ func TestAPIV2ParseTweetWithPoll(t *testing.T) { assert.Equal(1440*60, poll.VotingDuration) } +/** + * Parse a tweet with a Space + */ +func TestAPIV2ParseTweetWithSpace(t *testing.T) { + assert := assert.New(t) + data, err := os.ReadFile("test_responses/api_v2/tweet_with_audiospaces_card.json") + if err != nil { + panic(err) + } + + var tweet_result APIV2Result + err = json.Unmarshal(data, &tweet_result) + assert.NoError(err) + + trove := tweet_result.ToTweetTrove(true) + + assert.Len(trove.Tweets, 1) + tweet, ok := trove.Tweets[1497647006445146113] + require.True(t, ok) + assert.Len(tweet.Urls, 0) + assert.Len(tweet.Spaces, 1) + + s := tweet.Spaces[0] + assert.Equal(SpaceID("1dRJZlRNZDzKB"), s.ID) +} + func TestParseAPIV2UserFeed(t *testing.T) { data, err := os.ReadFile("test_responses/api_v2/user_feed_apiv2.json") if err != nil { diff --git a/scraper/test_responses/api_v2/tweet_with_audiospaces_card.json b/scraper/test_responses/api_v2/tweet_with_audiospaces_card.json new file mode 100644 index 0000000..5126efd --- /dev/null +++ b/scraper/test_responses/api_v2/tweet_with_audiospaces_card.json @@ -0,0 +1 @@ +{"result":{"__typename":"Tweet","rest_id":"1497647006445146113","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjoxMzQ1Mjk5Nzg1NDE5NTI2MTQ3","rest_id":"1345299785419526147","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Sat Jan 02 09:24:40 +0000 2021","default_profile":true,"default_profile_image":false,"description":"Amateur archivist of legitimist thought. Blogs at Carlsbad 1819.","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"carlsbad1819.wordpress.com","expanded_url":"https://carlsbad1819.wordpress.com","url":"https://t.co/oNR8vghQjo","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":18312,"followers_count":4679,"friends_count":154,"has_custom_timelines":false,"is_translator":false,"listed_count":25,"location":"","media_count":1562,"name":"Nigel Carlsbad 🇷🇺","normal_followers_count":4679,"pinned_tweet_ids_str":["1430589999767040000"],"profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":55.49,"rgb":{"blue":237,"green":237,"red":237}},{"percentage":31.76,"rgb":{"blue":78,"green":78,"red":78}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1479858746436243471/ra64x_1w_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"pompilivs","statuses_count":7752,"translator_type":"none","url":"https://t.co/oNR8vghQjo","verified":false,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"card":{"rest_id":"https://t.co/5RLbEwQgvH","legacy":{"binding_values":[{"key":"id","value":{"string_value":"1dRJZlRNZDzKB","type":"STRING"}},{"key":"card_url","value":{"scribe_key":"card_url","string_value":"https://t.co/5RLbEwQgvH","type":"STRING"}}],"card_platform":{"platform":{"audience":{"name":"production"},"device":{"name":"Swift","version":"12"}}},"name":"3691233323:audiospace","url":"https://t.co/5RLbEwQgvH","user_refs":[]}},"legacy":{"created_at":"Sat Feb 26 18:57:25 +0000 2022","conversation_id_str":"1497647006445146113","display_text_range":[0,32],"entities":{"user_mentions":[],"urls":[{"display_url":"twitter.com/i/spaces/1dRJZ…","expanded_url":"https://twitter.com/i/spaces/1dRJZlRNZDzKB","url":"https://t.co/5RLbEwQgvH","indices":[9,32]}],"hashtags":[],"symbols":[]},"favorite_count":44,"favorited":false,"full_text":"Starting https://t.co/5RLbEwQgvH","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":2,"reply_count":0,"retweet_count":8,"retweeted":false,"source":"Twitter for Android","user_id_str":"1345299785419526147","id_str":"1497647006445146113"}}}