Implement user feed using cursors

pkg/persistence/compound_queries_test.go

@@ -18,7 +18,10 @@ func TestBuildUserFeed(t *testing.T) {
 	profile, err := persistence.LoadProfile("../../sample_data/profile")
 	require.NoError(err)
 
-	feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(0))
+	c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
+	c.PageSize = 2
+
+	feed, err := profile.NextPage(c)
 	require.NoError(err)
 
 	assert.Len(feed.Retweets, 2)
@@ -45,7 +48,7 @@ func TestBuildUserFeed(t *testing.T) {
 	assert.Equal(feed.Items[1].TweetID, TweetID(1490116725395927042))
 	assert.Equal(feed.Items[1].RetweetID, TweetID(1490119308692766723))
 
-	assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1644107102))
+	assert.Equal(feed.CursorBottom.CursorValue, 1644107102)
 }
 
 // Should load a feed in the middle (i.e., after some timestamp)
@@ -56,7 +59,11 @@ func TestBuildUserFeedPage2(t *testing.T) {
 	profile, err := persistence.LoadProfile("../../sample_data/profile")
 	require.NoError(err)
 
-	feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1644107102))
+	c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
+	c.PageSize = 2
+	c.CursorPosition = persistence.CURSOR_MIDDLE
+	c.CursorValue = 1644107102
+	feed, err := profile.NextPage(c)
 	require.NoError(err)
 
 	assert.Len(feed.Retweets, 1)
@@ -81,7 +88,7 @@ func TestBuildUserFeedPage2(t *testing.T) {
 	assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744))
 	assert.Equal(feed.Items[1].RetweetID, TweetID(0))
 
-	assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1635367140))
+	assert.Equal(feed.CursorBottom.CursorValue, 1635367140)
 }
 
 // When the end of the feed is reached, an "End of feed" error should be raised
@@ -92,14 +99,19 @@ func TestBuildUserFeedEnd(t *testing.T) {
 	profile, err := persistence.LoadProfile("../../sample_data/profile")
 	require.NoError(err)
 
-	feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1)) // Won't be anything after "1"
+	c := persistence.NewUserFeedCursor(UserHandle("cernovich"))
-	require.Error(err)
+	c.PageSize = 2
-	require.ErrorIs(err, persistence.ErrEndOfFeed)
+	c.CursorPosition = persistence.CURSOR_MIDDLE
+	c.CursorValue = 1 // Won't be anything
+	feed, err := profile.NextPage(c)
+	require.NoError(err)
 
 	assert.Len(feed.Retweets, 0)
 	assert.Len(feed.Tweets, 0)
 	assert.Len(feed.Users, 0)
 	require.Len(feed.Items, 0)
+
+	assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END)
 }
 
 func TestTweetDetailWithReplies(t *testing.T) {

pkg/persistence/compound_ssf_queries.go

@@ -101,8 +101,9 @@ type Cursor struct {
 	// Search params
 	Keywords              []string
 	FromUserHandle        scraper.UserHandle
-	ToUserHandles         []scraper.UserHandle
 	RetweetedByUserHandle scraper.UserHandle
+	ByUserHandle          scraper.UserHandle
+	ToUserHandles         []scraper.UserHandle
 	SinceTimestamp        scraper.Timestamp
 	UntilTimestamp        scraper.Timestamp
 	FilterLinks           Filter
@@ -114,6 +115,7 @@ type Cursor struct {
 	FilterOfflineFollowed Filter
 }
 
+// Generate a cursor with some reasonable defaults
 func NewCursor() Cursor {
 	return Cursor{
 		Keywords:       []string{},
@@ -129,6 +131,7 @@ func NewCursor() Cursor {
 	}
 }
 
+// Generate a cursor appropriate for fetching the Offline Timeline
 func NewTimelineCursor() Cursor {
 	return Cursor{
 		Keywords:       []string{},
@@ -144,6 +147,22 @@ func NewTimelineCursor() Cursor {
 	}
 }
 
+// Generate a cursor appropriate for fetching a User Feed
+func NewUserFeedCursor(h scraper.UserHandle) Cursor {
+	return Cursor{
+		Keywords:       []string{},
+		ToUserHandles:  []scraper.UserHandle{},
+		SinceTimestamp: scraper.TimestampFromUnix(0),
+		UntilTimestamp: scraper.TimestampFromUnix(0),
+		CursorPosition: CURSOR_START,
+		CursorValue:    0,
+		SortOrder:      SORT_ORDER_NEWEST,
+		PageSize:       50,
+
+		ByUserHandle: h,
+	}
+}
+
 func (p Profile) NextPage(c Cursor) (Feed, error) {
 	where_clauses := []string{}
 	bind_values := []interface{}{}
@@ -154,7 +173,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
 		bind_values = append(bind_values, fmt.Sprintf("%%%s%%", kw))
 	}
 
-	// From, to, and RT'd by user handles
+	// From, to, by, and RT'd by user handles
 	if c.FromUserHandle != "" {
 		where_clauses = append(where_clauses, "user_id = (select id from users where handle like ?)")
 		bind_values = append(bind_values, c.FromUserHandle)
@@ -167,6 +186,10 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
 		where_clauses = append(where_clauses, "retweeted_by = (select id from users where handle like ?)")
 		bind_values = append(bind_values, c.RetweetedByUserHandle)
 	}
+	if c.ByUserHandle != "" {
+		where_clauses = append(where_clauses, "by_user_id = (select id from users where handle like ?)")
+		bind_values = append(bind_values, c.ByUserHandle)
+	}
 
 	// Since and until timestamps
 	if c.SinceTimestamp.Unix() != 0 {