me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement user feed using cursors

Browse Source
This commit is contained in:
Alessio 2023-08-12 10:02:51 -03:00
parent 5568a86651
commit 17084408f7
2 changed files with 44 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
pkg/persistence/compound_queries_test.go
View File

@ -18,7 +18,10 @@ func TestBuildUserFeed(t *testing.T) {
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(0))
c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
c.PageSize = 2
feed, err := profile.NextPage(c)
require.NoError(err)
assert.Len(feed.Retweets, 2)
@ -45,7 +48,7 @@ func TestBuildUserFeed(t *testing.T) {
assert.Equal(feed.Items[1].TweetID, TweetID(1490116725395927042))
assert.Equal(feed.Items[1].RetweetID, TweetID(1490119308692766723))
assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1644107102))
assert.Equal(feed.CursorBottom.CursorValue, 1644107102)
}
// Should load a feed in the middle (i.e., after some timestamp)
@ -56,7 +59,11 @@ func TestBuildUserFeedPage2(t *testing.T) {
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1644107102))
c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
c.PageSize = 2
c.CursorPosition = persistence.CURSOR_MIDDLE
c.CursorValue = 1644107102
feed, err := profile.NextPage(c)
require.NoError(err)
assert.Len(feed.Retweets, 1)
@ -81,7 +88,7 @@ func TestBuildUserFeedPage2(t *testing.T) {
assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744))
assert.Equal(feed.Items[1].RetweetID, TweetID(0))
assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1635367140))
assert.Equal(feed.CursorBottom.CursorValue, 1635367140)
}
// When the end of the feed is reached, an "End of feed" error should be raised
@ -92,14 +99,19 @@ func TestBuildUserFeedEnd(t *testing.T) {
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1)) // Won't be anything after "1"
require.Error(err)
require.ErrorIs(err, persistence.ErrEndOfFeed)
c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
c.PageSize = 2
c.CursorPosition = persistence.CURSOR_MIDDLE
c.CursorValue = 1 // Won't be anything
feed, err := profile.NextPage(c)
require.NoError(err)
assert.Len(feed.Retweets, 0)
assert.Len(feed.Tweets, 0)
assert.Len(feed.Users, 0)
require.Len(feed.Items, 0)
assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END)
}
func TestTweetDetailWithReplies(t *testing.T) {

27
pkg/persistence/compound_ssf_queries.go
View File

@ -101,8 +101,9 @@ type Cursor struct {
// Search params
Keywords []string
FromUserHandle scraper.UserHandle
ToUserHandles []scraper.UserHandle
RetweetedByUserHandle scraper.UserHandle
ByUserHandle scraper.UserHandle
ToUserHandles []scraper.UserHandle
SinceTimestamp scraper.Timestamp
UntilTimestamp scraper.Timestamp
FilterLinks Filter
@ -114,6 +115,7 @@ type Cursor struct {
FilterOfflineFollowed Filter
}
// Generate a cursor with some reasonable defaults
func NewCursor() Cursor {
return Cursor{
Keywords: []string{},
@ -129,6 +131,7 @@ func NewCursor() Cursor {
}
}
// Generate a cursor appropriate for fetching the Offline Timeline
func NewTimelineCursor() Cursor {
return Cursor{
Keywords: []string{},
@ -144,6 +147,22 @@ func NewTimelineCursor() Cursor {
}
}
// Generate a cursor appropriate for fetching a User Feed
func NewUserFeedCursor(h scraper.UserHandle) Cursor {
return Cursor{
Keywords: []string{},
ToUserHandles: []scraper.UserHandle{},
SinceTimestamp: scraper.TimestampFromUnix(0),
UntilTimestamp: scraper.TimestampFromUnix(0),
CursorPosition: CURSOR_START,
CursorValue: 0,
SortOrder: SORT_ORDER_NEWEST,
PageSize: 50,
ByUserHandle: h,
}
}
func (p Profile) NextPage(c Cursor) (Feed, error) {
where_clauses := []string{}
bind_values := []interface{}{}
@ -154,7 +173,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
bind_values = append(bind_values, fmt.Sprintf("%%%s%%", kw))
}
// From, to, and RT'd by user handles
// From, to, by, and RT'd by user handles
if c.FromUserHandle != "" {
where_clauses = append(where_clauses, "user_id = (select id from users where handle like ?)")
bind_values = append(bind_values, c.FromUserHandle)
@ -167,6 +186,10 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
where_clauses = append(where_clauses, "retweeted_by = (select id from users where handle like ?)")
bind_values = append(bind_values, c.RetweetedByUserHandle)
}
if c.ByUserHandle != "" {
where_clauses = append(where_clauses, "by_user_id = (select id from users where handle like ?)")
bind_values = append(bind_values, c.ByUserHandle)
}
// Since and until timestamps
if c.SinceTimestamp.Unix() != 0 {