me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix parsing of other media items in DMs as 'urls'

Browse Source
This commit is contained in:
Alessio 2024-03-09 15:11:33 -08:00
parent 9fa0cbf6d6
commit 07233a7153
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/scraper/api_types_dms_test.go
View File

@ -71,6 +71,7 @@ func TestParseAPIDMMessageWithEmbeddedTweet(t *testing.T) {
assert.True(is_ok) assert.True(is_ok)
assert.Equal(m.EmbeddedTweetID, TweetID(1665509126737129472)) assert.Equal(m.EmbeddedTweetID, TweetID(1665509126737129472))
assert.Equal(m.Text, "Check this out") assert.Equal(m.Text, "Check this out")
assert.Len(m.Urls, 0) // Embedded tweets should not parse as URLs
assert.Len(trove.Tweets, 1) assert.Len(trove.Tweets, 1)
_, is_ok = trove.Tweets[TweetID(1665509126737129472)] _, is_ok = trove.Tweets[TweetID(1665509126737129472)]
@ -96,6 +97,7 @@ func TestParseAPIDMMessageWithEmbeddedImage(t *testing.T) {
assert.Len(trove.Messages, 1) assert.Len(trove.Messages, 1)
m, is_ok := trove.Messages[DMMessageID(1766224476729995648)] m, is_ok := trove.Messages[DMMessageID(1766224476729995648)]
assert.True(is_ok) assert.True(is_ok)
assert.Len(m.Urls, 0) // Embedded images should not parse as URLs
// Check that the short-URL is stripped // Check that the short-URL is stripped
assert.Equal("A gastropub staffed by white college girls and the chefs are all Latino", m.Text) assert.Equal("A gastropub staffed by white college girls and the chefs are all Latino", m.Text)
@ -120,6 +122,7 @@ func TestParseAPIDMMessageWithEmbeddedVideo(t *testing.T) {
assert.Len(trove.Messages, 1) assert.Len(trove.Messages, 1)
m, is_ok := trove.Messages[DMMessageID(1766248283901776125)] m, is_ok := trove.Messages[DMMessageID(1766248283901776125)]
assert.True(is_ok) assert.True(is_ok)
assert.Len(m.Urls, 0) // Embedded videos should not parse as URLs
// Check the short-URL is stripped // Check the short-URL is stripped
assert.Equal("", m.Text) assert.Equal("", m.Text)

14
pkg/scraper/dm_message.go
View File

@ -67,6 +67,20 @@ func ParseAPIDMMessage(message APIDMMessage) DMMessage {
// Process URLs and link previews // Process URLs and link previews
for _, url := range message.MessageData.Entities.URLs { for _, url := range message.MessageData.Entities.URLs {
// Skip it if it's an embedded tweet
_, id, is_ok := TryParseTweetUrl(url.ExpandedURL)
if is_ok && id == TweetID(message.MessageData.Attachment.Tweet.Status.ID) {
continue
}
// Skip it if it's an embedded image
if message.MessageData.Attachment.Photo.URL == url.ShortenedUrl {
continue
}
// Skip it if it's an embedded video
if message.MessageData.Attachment.Video.URL == url.ShortenedUrl {
continue
}
var new_url Url var new_url Url
if message.MessageData.Attachment.Card.ShortenedUrl == url.ShortenedUrl { if message.MessageData.Attachment.Card.ShortenedUrl == url.ShortenedUrl {
if message.MessageData.Attachment.Card.Name == "3691233323:audiospace" { if message.MessageData.Attachment.Card.Name == "3691233323:audiospace" {