me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add search filters for 'media' (union of video and image) and negative search filters (prefixed with '-')

Browse Source
This commit is contained in:
Alessio 2023-10-12 10:30:33 -07:00
parent 9bb751afa5
commit 9aff2b8a93
2 changed files with 48 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/persistence/compound_ssf_queries.go
View File

@ -116,6 +116,7 @@ type Cursor struct {
FilterLinks Filter
FilterImages Filter
FilterVideos Filter
FilterMedia Filter
FilterPolls Filter
FilterSpaces Filter
FilterReplies Filter
@ -266,12 +267,30 @@ func (c *Cursor) apply_token(token string) error {
c.FilterImages = REQUIRE
case "videos":
c.FilterVideos = REQUIRE
case "media":
c.FilterMedia = REQUIRE
case "polls":
c.FilterPolls = REQUIRE
case "spaces":
c.FilterSpaces = REQUIRE
}
case "-filter":
switch parts[1] {
case "links":
c.FilterLinks = EXCLUDE
case "images":
c.FilterImages = EXCLUDE
case "videos":
c.FilterVideos = EXCLUDE
case "media":
c.FilterMedia = EXCLUDE
case "polls":
c.FilterPolls = EXCLUDE
case "spaces":
c.FilterSpaces = EXCLUDE
}
}
if err != nil {
return fmt.Errorf("query token %q: %w", token, ErrInvalidQuery)
}
@ -335,6 +354,14 @@ func (p Profile) NextPage(c Cursor, current_user_id scraper.UserID) (Feed, error
case EXCLUDE:
where_clauses = append(where_clauses, "not exists (select 1 from videos where videos.tweet_id = tweets.id)")
}
switch c.FilterMedia {
case REQUIRE:
where_clauses = append(where_clauses, `(exists (select 1 from videos where videos.tweet_id = tweets.id)
or exists (select 1 from images where images.tweet_id = tweets.id))`)
case EXCLUDE:
where_clauses = append(where_clauses, `not (exists (select 1 from videos where videos.tweet_id = tweets.id)
or exists (select 1 from images where images.tweet_id = tweets.id))`)
}
switch c.FilterPolls {
case REQUIRE:
where_clauses = append(where_clauses, "exists (select 1 from polls where polls.tweet_id = tweets.id)")

21
pkg/persistence/compound_ssf_queries_test.go
View File

@ -250,6 +250,18 @@ func TestSearchMediaFilters(t *testing.T) {
assert.Equal(feed.Items[0].TweetID, TweetID(1426619468327882761))
assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744))
// Media (generic)
c = persistence.NewCursor()
c.SortOrder = persistence.SORT_ORDER_MOST_LIKES
c.FilterMedia = persistence.REQUIRE
feed, err = profile.NextPage(c, UserID(0))
require.NoError(err)
assert.Len(feed.Items, 4)
assert.Equal(feed.Items[0].TweetID, TweetID(1426619468327882761))
assert.Equal(feed.Items[1].TweetID, TweetID(1261483383483293700))
assert.Equal(feed.Items[2].TweetID, TweetID(1426669666928414720))
assert.Equal(feed.Items[3].TweetID, TweetID(1453461248142495744))
// Polls
c = persistence.NewCursor()
c.FilterPolls = persistence.REQUIRE
@ -265,4 +277,13 @@ func TestSearchMediaFilters(t *testing.T) {
require.NoError(err)
assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1624833173514293249))
// Negative filter (images)
c = persistence.NewCursor()
c.FilterImages = persistence.EXCLUDE
c.FromUserHandle = UserHandle("covfefeanon")
feed, err = profile.NextPage(c, UserID(0))
require.NoError(err)
assert.Len(feed.Items, 1)
assert.Equal(feed.Items[0].TweetID, TweetID(1428951883058753537))
}