diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index 11d28e7..bb276ca 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -18,7 +18,10 @@ func TestBuildUserFeed(t *testing.T) { profile, err := persistence.LoadProfile("../../sample_data/profile") require.NoError(err) - feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(0)) + c := persistence.NewUserFeedCursor(UserHandle("cernovich")) + c.PageSize = 2 + + feed, err := profile.NextPage(c) require.NoError(err) assert.Len(feed.Retweets, 2) @@ -45,7 +48,7 @@ func TestBuildUserFeed(t *testing.T) { assert.Equal(feed.Items[1].TweetID, TweetID(1490116725395927042)) assert.Equal(feed.Items[1].RetweetID, TweetID(1490119308692766723)) - assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1644107102)) + assert.Equal(feed.CursorBottom.CursorValue, 1644107102) } // Should load a feed in the middle (i.e., after some timestamp) @@ -56,7 +59,11 @@ func TestBuildUserFeedPage2(t *testing.T) { profile, err := persistence.LoadProfile("../../sample_data/profile") require.NoError(err) - feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1644107102)) + c := persistence.NewUserFeedCursor(UserHandle("cernovich")) + c.PageSize = 2 + c.CursorPosition = persistence.CURSOR_MIDDLE + c.CursorValue = 1644107102 + feed, err := profile.NextPage(c) require.NoError(err) assert.Len(feed.Retweets, 1) @@ -81,7 +88,7 @@ func TestBuildUserFeedPage2(t *testing.T) { assert.Equal(feed.Items[1].TweetID, TweetID(1453461248142495744)) assert.Equal(feed.Items[1].RetweetID, TweetID(0)) - assert.Equal(feed.BottomTimestamp(), TimestampFromUnix(1635367140)) + assert.Equal(feed.CursorBottom.CursorValue, 1635367140) } // When the end of the feed is reached, an "End of feed" error should be raised @@ -92,14 +99,19 @@ func TestBuildUserFeedEnd(t *testing.T) { profile, err := persistence.LoadProfile("../../sample_data/profile") require.NoError(err) - feed, err := profile.GetUserFeed(358545917, 2, TimestampFromUnix(1)) // Won't be anything after "1" - require.Error(err) - require.ErrorIs(err, persistence.ErrEndOfFeed) + c := persistence.NewUserFeedCursor(UserHandle("cernovich")) + c.PageSize = 2 + c.CursorPosition = persistence.CURSOR_MIDDLE + c.CursorValue = 1 // Won't be anything + feed, err := profile.NextPage(c) + require.NoError(err) assert.Len(feed.Retweets, 0) assert.Len(feed.Tweets, 0) assert.Len(feed.Users, 0) require.Len(feed.Items, 0) + + assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END) } func TestTweetDetailWithReplies(t *testing.T) { diff --git a/pkg/persistence/compound_ssf_queries.go b/pkg/persistence/compound_ssf_queries.go index a30c92f..15e4b92 100644 --- a/pkg/persistence/compound_ssf_queries.go +++ b/pkg/persistence/compound_ssf_queries.go @@ -101,8 +101,9 @@ type Cursor struct { // Search params Keywords []string FromUserHandle scraper.UserHandle - ToUserHandles []scraper.UserHandle RetweetedByUserHandle scraper.UserHandle + ByUserHandle scraper.UserHandle + ToUserHandles []scraper.UserHandle SinceTimestamp scraper.Timestamp UntilTimestamp scraper.Timestamp FilterLinks Filter @@ -114,6 +115,7 @@ type Cursor struct { FilterOfflineFollowed Filter } +// Generate a cursor with some reasonable defaults func NewCursor() Cursor { return Cursor{ Keywords: []string{}, @@ -129,6 +131,7 @@ func NewCursor() Cursor { } } +// Generate a cursor appropriate for fetching the Offline Timeline func NewTimelineCursor() Cursor { return Cursor{ Keywords: []string{}, @@ -144,6 +147,22 @@ func NewTimelineCursor() Cursor { } } +// Generate a cursor appropriate for fetching a User Feed +func NewUserFeedCursor(h scraper.UserHandle) Cursor { + return Cursor{ + Keywords: []string{}, + ToUserHandles: []scraper.UserHandle{}, + SinceTimestamp: scraper.TimestampFromUnix(0), + UntilTimestamp: scraper.TimestampFromUnix(0), + CursorPosition: CURSOR_START, + CursorValue: 0, + SortOrder: SORT_ORDER_NEWEST, + PageSize: 50, + + ByUserHandle: h, + } +} + func (p Profile) NextPage(c Cursor) (Feed, error) { where_clauses := []string{} bind_values := []interface{}{} @@ -154,7 +173,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { bind_values = append(bind_values, fmt.Sprintf("%%%s%%", kw)) } - // From, to, and RT'd by user handles + // From, to, by, and RT'd by user handles if c.FromUserHandle != "" { where_clauses = append(where_clauses, "user_id = (select id from users where handle like ?)") bind_values = append(bind_values, c.FromUserHandle) @@ -167,6 +186,10 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { where_clauses = append(where_clauses, "retweeted_by = (select id from users where handle like ?)") bind_values = append(bind_values, c.RetweetedByUserHandle) } + if c.ByUserHandle != "" { + where_clauses = append(where_clauses, "by_user_id = (select id from users where handle like ?)") + bind_values = append(bind_values, c.ByUserHandle) + } // Since and until timestamps if c.SinceTimestamp.Unix() != 0 {