From ac763a97a63a3836f4d0bfc1c1cb6ffa5ef82e48 Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 3 Feb 2025 16:25:34 -0800 Subject: [PATCH] REFACTOR: in persistence package, import scraper as '.' --- persistence_test_sqlite_memory.sh | 2 +- pkg/persistence/bookmark_queries.go | 10 +-- pkg/persistence/compound_ssf_queries.go | 98 ++++++++++++------------- pkg/persistence/likes_queries.go | 10 +-- pkg/persistence/media_download.go | 26 +++---- pkg/persistence/media_download_test.go | 6 +- pkg/persistence/media_queries.go | 18 ++--- pkg/persistence/media_queries_test.go | 14 ++-- pkg/persistence/notification_queries.go | 1 + pkg/persistence/profile.go | 8 +- pkg/persistence/retweet_queries.go | 8 +- pkg/persistence/session.go | 10 ++- pkg/persistence/session_test.go | 6 +- pkg/persistence/space_queries.go | 15 ++-- pkg/persistence/space_queries_test.go | 29 ++++---- pkg/persistence/tweet_queries_test.go | 12 +-- pkg/persistence/tweet_trove_queries.go | 4 +- pkg/persistence/user_queries.go | 36 ++++----- pkg/persistence/user_queries_test.go | 44 +++++------ pkg/persistence/versions_test.go | 4 +- 20 files changed, 185 insertions(+), 176 deletions(-) diff --git a/persistence_test_sqlite_memory.sh b/persistence_test_sqlite_memory.sh index b92d253..490d7fe 100755 --- a/persistence_test_sqlite_memory.sh +++ b/persistence_test_sqlite_memory.sh @@ -1,5 +1,5 @@ #!/bin/bash sudo mount -t tmpfs -o size=100M tmpfs pkg/persistence/test_profiles -sudo mount -t tmpfs -o size=100M tmpfs cmd/data +sudo mount -t tmpfs -o size=500M tmpfs cmd/data sudo mount -t tmpfs -o size=1000M tmpfs sample_data/profile diff --git a/pkg/persistence/bookmark_queries.go b/pkg/persistence/bookmark_queries.go index 7e79f02..429aef0 100644 --- a/pkg/persistence/bookmark_queries.go +++ b/pkg/persistence/bookmark_queries.go @@ -3,10 +3,10 @@ package persistence import ( "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) -func (p Profile) SaveBookmark(l scraper.Bookmark) error { +func (p Profile) SaveBookmark(l Bookmark) error { _, err := p.DB.NamedExec(` insert into bookmarks (sort_order, user_id, tweet_id) values (:sort_order, :user_id, :tweet_id) @@ -20,7 +20,7 @@ func (p Profile) SaveBookmark(l scraper.Bookmark) error { return nil } -func (p Profile) DeleteBookmark(l scraper.Bookmark) error { +func (p Profile) DeleteBookmark(l Bookmark) error { _, err := p.DB.NamedExec(`delete from bookmarks where user_id = :user_id and tweet_id = :tweet_id`, l) if err != nil { return fmt.Errorf("Error executing DeleteBookmark(%#v):\n %w", l, err) @@ -28,8 +28,8 @@ func (p Profile) DeleteBookmark(l scraper.Bookmark) error { return nil } -func (p Profile) GetBookmarkBySortID(id scraper.BookmarkSortID) (scraper.Bookmark, error) { - var l scraper.Bookmark +func (p Profile) GetBookmarkBySortID(id BookmarkSortID) (Bookmark, error) { + var l Bookmark err := p.DB.Get(&l, ` select sort_order, user_id, tweet_id from bookmarks diff --git a/pkg/persistence/compound_ssf_queries.go b/pkg/persistence/compound_ssf_queries.go index da49461..6325923 100644 --- a/pkg/persistence/compound_ssf_queries.go +++ b/pkg/persistence/compound_ssf_queries.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type SortOrder int @@ -91,7 +91,7 @@ func (o SortOrder) NextCursorValue(r CursorResult) int { panic(fmt.Sprintf("Invalid sort order: %d", o)) } } -func (o SortOrder) NextDMCursorValue(m scraper.DMMessage) int64 { +func (o SortOrder) NextDMCursorValue(m DMMessage) int64 { switch o { case SORT_ORDER_NEWEST, SORT_ORDER_OLDEST: return m.SentAt.UnixMilli() @@ -131,12 +131,12 @@ const ( ) type CursorResult struct { - scraper.Tweet - scraper.Retweet + Tweet + Retweet Chrono int `db:"chrono"` LikeSortOrder int `db:"likes_sort_order"` BookmarkSortOrder int `db:"bookmarks_sort_order"` - ByUserID scraper.UserID `db:"by_user_id"` + ByUserID UserID `db:"by_user_id"` } type Cursor struct { @@ -147,16 +147,16 @@ type Cursor struct { // Search params Keywords []string - FromUserHandle scraper.UserHandle // Tweeted by this user - RetweetedByUserHandle scraper.UserHandle // Retweeted by this user - ByUserHandle scraper.UserHandle // Either tweeted or retweeted by this user - ToUserHandles []scraper.UserHandle // In reply to these users - LikedByUserHandle scraper.UserHandle // Liked by this user - BookmarkedByUserHandle scraper.UserHandle // Bookmarked by this user - ListID scraper.ListID // Either tweeted or retweeted by users from this List - FollowedByUserHandle scraper.UserHandle // Either tweeted or retweeted by users followed by this user - SinceTimestamp scraper.Timestamp - UntilTimestamp scraper.Timestamp + FromUserHandle UserHandle // Tweeted by this user + RetweetedByUserHandle UserHandle // Retweeted by this user + ByUserHandle UserHandle // Either tweeted or retweeted by this user + ToUserHandles []UserHandle // In reply to these users + LikedByUserHandle UserHandle // Liked by this user + BookmarkedByUserHandle UserHandle // Bookmarked by this user + ListID ListID // Either tweeted or retweeted by users from this List + FollowedByUserHandle UserHandle // Either tweeted or retweeted by users followed by this user + SinceTimestamp Timestamp + UntilTimestamp Timestamp TombstoneType string FilterLinks Filter FilterImages Filter @@ -173,9 +173,9 @@ type Cursor struct { func NewCursor() Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_NEWEST, @@ -189,9 +189,9 @@ func NewCursor() Cursor { func NewTimelineCursor() Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_NEWEST, @@ -202,13 +202,13 @@ func NewTimelineCursor() Cursor { } // Generate a cursor appropriate for showing a List feed -func NewListCursor(list_id scraper.ListID) Cursor { +func NewListCursor(list_id ListID) Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, + ToUserHandles: []UserHandle{}, ListID: list_id, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_NEWEST, @@ -217,12 +217,12 @@ func NewListCursor(list_id scraper.ListID) Cursor { } // Generate a cursor appropriate for fetching a User Feed -func NewUserFeedCursor(h scraper.UserHandle) Cursor { +func NewUserFeedCursor(h UserHandle) Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_NEWEST, @@ -233,12 +233,12 @@ func NewUserFeedCursor(h scraper.UserHandle) Cursor { } // Generate a cursor appropriate for a user's Media tab -func NewUserFeedMediaCursor(h scraper.UserHandle) Cursor { +func NewUserFeedMediaCursor(h UserHandle) Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_NEWEST, @@ -250,12 +250,12 @@ func NewUserFeedMediaCursor(h scraper.UserHandle) Cursor { } // Generate a cursor for a User's Likes -func NewUserFeedLikesCursor(h scraper.UserHandle) Cursor { +func NewUserFeedLikesCursor(h UserHandle) Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_LIKED_AT, @@ -266,12 +266,12 @@ func NewUserFeedLikesCursor(h scraper.UserHandle) Cursor { } // Generate a cursor for a User's Bookmarks -func NewUserFeedBookmarksCursor(h scraper.UserHandle) Cursor { +func NewUserFeedBookmarksCursor(h UserHandle) Cursor { return Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), CursorPosition: CURSOR_START, CursorValue: 0, SortOrder: SORT_ORDER_BOOKMARKED_AT, @@ -343,24 +343,24 @@ func (c *Cursor) apply_token(token string) error { var err error switch parts[0] { case "from": - c.FromUserHandle = scraper.UserHandle(parts[1]) + c.FromUserHandle = UserHandle(parts[1]) case "to": - c.ToUserHandles = append(c.ToUserHandles, scraper.UserHandle(parts[1])) + c.ToUserHandles = append(c.ToUserHandles, UserHandle(parts[1])) case "retweeted_by": - c.RetweetedByUserHandle = scraper.UserHandle(parts[1]) + c.RetweetedByUserHandle = UserHandle(parts[1]) c.FilterRetweets = NONE // Clear the "exclude retweets" filter set by default in NewCursor case "liked_by": - c.LikedByUserHandle = scraper.UserHandle(parts[1]) + c.LikedByUserHandle = UserHandle(parts[1]) case "bookmarked_by": - c.BookmarkedByUserHandle = scraper.UserHandle(parts[1]) + c.BookmarkedByUserHandle = UserHandle(parts[1]) case "followed_by": - c.FollowedByUserHandle = scraper.UserHandle(parts[1]) + c.FollowedByUserHandle = UserHandle(parts[1]) case "list": i, err := strconv.Atoi(parts[1]) if err != nil { return fmt.Errorf("%w: filter 'list:' must be a number (list ID), got %q", ErrInvalidQuery, parts[1]) } - c.ListID = scraper.ListID(i) + c.ListID = ListID(i) case "since": c.SinceTimestamp.Time, err = time.Parse("2006-01-02", parts[1]) case "until": @@ -413,7 +413,7 @@ func (c *Cursor) apply_token(token string) error { return nil } -func (p Profile) NextPage(c Cursor, current_user_id scraper.UserID) (Feed, error) { +func (p Profile) NextPage(c Cursor, current_user_id UserID) (Feed, error) { where_clauses := []string{} bind_values := []interface{}{} diff --git a/pkg/persistence/likes_queries.go b/pkg/persistence/likes_queries.go index 3664d2c..e8bb781 100644 --- a/pkg/persistence/likes_queries.go +++ b/pkg/persistence/likes_queries.go @@ -3,10 +3,10 @@ package persistence import ( "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) -func (p Profile) SaveLike(l scraper.Like) error { +func (p Profile) SaveLike(l Like) error { _, err := p.DB.NamedExec(` insert into likes (sort_order, user_id, tweet_id) values (:sort_order, :user_id, :tweet_id) @@ -20,7 +20,7 @@ func (p Profile) SaveLike(l scraper.Like) error { return nil } -func (p Profile) DeleteLike(l scraper.Like) error { +func (p Profile) DeleteLike(l Like) error { _, err := p.DB.NamedExec(`delete from likes where user_id = :user_id and tweet_id = :tweet_id`, l) if err != nil { return fmt.Errorf("Error executing DeleteLike(%#v):\n %w", l, err) @@ -28,8 +28,8 @@ func (p Profile) DeleteLike(l scraper.Like) error { return nil } -func (p Profile) GetLikeBySortID(id scraper.LikeSortID) (scraper.Like, error) { - var l scraper.Like +func (p Profile) GetLikeBySortID(id LikeSortID) (Like, error) { + var l Like err := p.DB.Get(&l, ` select sort_order, user_id, tweet_id from likes diff --git a/pkg/persistence/media_download.go b/pkg/persistence/media_download.go index bdeec97..5169e1e 100644 --- a/pkg/persistence/media_download.go +++ b/pkg/persistence/media_download.go @@ -6,7 +6,7 @@ import ( "os" "path" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type MediaDownloader interface { @@ -14,7 +14,7 @@ type MediaDownloader interface { } type DefaultDownloader struct { - *scraper.API + *API } // Download a file over HTTP and save it. @@ -47,7 +47,7 @@ func (d DefaultDownloader) Curl(url string, outpath string) error { // Downloads an Image, and if successful, marks it as downloaded in the DB // DUPE: download-image -func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownloader) error { +func (p Profile) download_tweet_image(img *Image, downloader MediaDownloader) error { outfile := path.Join(p.ProfileDir, "images", img.LocalFilename) err := downloader.Curl(img.RemoteURL, outfile) if err != nil { @@ -59,12 +59,12 @@ func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownlo // Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB // DUPE: download-video -func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownloader) error { +func (p Profile) download_tweet_video(v *Video, downloader MediaDownloader) error { // Download the video outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename) err := downloader.Curl(v.RemoteURL, outfile) - if errors.Is(err, scraper.ErrorDMCA) { + if errors.Is(err, ErrorDMCA) { v.IsDownloaded = false v.IsBlockedByDMCA = true } else if err != nil { @@ -86,7 +86,7 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload // Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB // DUPE: download-link-thumbnail -func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error { +func (p Profile) download_link_thumbnail(url *Url, downloader MediaDownloader) error { if url.HasCard && url.HasThumbnail { outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) err := downloader.Curl(url.ThumbnailRemoteUrl, outfile) @@ -100,12 +100,12 @@ func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownl // Download a tweet's video and picture content. // Wraps the `DownloadTweetContentWithInjector` method with the default (i.e., real) downloader. -func (p Profile) DownloadTweetContentFor(t *scraper.Tweet, api *scraper.API) error { +func (p Profile) DownloadTweetContentFor(t *Tweet, api *API) error { return p.DownloadTweetContentWithInjector(t, DefaultDownloader{API: api}) } // Enable injecting a custom MediaDownloader (i.e., for testing) -func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader MediaDownloader) error { +func (p Profile) DownloadTweetContentWithInjector(t *Tweet, downloader MediaDownloader) error { // Check if content needs to be downloaded; if not, just return if !p.CheckTweetContentDownloadNeeded(*t) { return nil @@ -141,12 +141,12 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M } // Download a user's banner and profile images -func (p Profile) DownloadUserContentFor(u *scraper.User, api *scraper.API) error { +func (p Profile) DownloadUserContentFor(u *User, api *API) error { return p.DownloadUserContentWithInjector(u, DefaultDownloader{API: api}) } // Enable injecting a custom MediaDownloader (i.e., for testing) -func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error { +func (p Profile) DownloadUserContentWithInjector(u *User, downloader MediaDownloader) error { if !p.CheckUserContentDownloadNeeded(*u) { return nil } @@ -155,7 +155,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med var target_url string if u.ProfileImageUrl == "" { - target_url = scraper.DEFAULT_PROFILE_IMAGE_URL + target_url = DEFAULT_PROFILE_IMAGE_URL } else { target_url = u.ProfileImageUrl } @@ -170,7 +170,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med outfile = p.get_banner_image_output_path(*u) err = downloader.Curl(u.BannerImageUrl, outfile) - if errors.Is(err, scraper.ErrMediaDownload404) { + if errors.Is(err, ErrMediaDownload404) { // Try adding "600x200". Not sure why this does this but sometimes it does. err = downloader.Curl(u.BannerImageUrl+"/600x200", outfile) } @@ -186,7 +186,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med // Download a User's tiny profile image, if it hasn't been downloaded yet. // If it has been downloaded, do nothing. // If this user should have a big profile picture, defer to the regular `DownloadUserContentFor` method. -func (p Profile) DownloadUserProfileImageTiny(u *scraper.User, api *scraper.API) error { +func (p Profile) DownloadUserProfileImageTiny(u *User, api *API) error { if p.IsFollowing(*u) { return p.DownloadUserContentFor(u, api) } diff --git a/pkg/persistence/media_download_test.go b/pkg/persistence/media_download_test.go index 99ff65e..01e0123 100644 --- a/pkg/persistence/media_download_test.go +++ b/pkg/persistence/media_download_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Some types to spy on a MediaDownloader @@ -38,7 +38,7 @@ func (d FakeDownloader) Contains(result SpyResult) bool { return false } -func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) { +func test_all_downloaded(tweet Tweet, yes_or_no bool, t *testing.T) { error_msg := map[bool]string{ true: "Expected to be downloaded, but it wasn't", false: "Expected not to be downloaded, but it was", @@ -147,7 +147,7 @@ func TestDownloadDefaultUserContent(t *testing.T) { // Check that the downloader was called with the appropriate stuff assert.Len(*fake_downloader.Spy, 1) assert.True(fake_downloader.Contains(SpyResult{ - scraper.DEFAULT_PROFILE_IMAGE_URL, + DEFAULT_PROFILE_IMAGE_URL, "test_profiles/TestMediaQueries/profile_images/default_profile.png", })) } diff --git a/pkg/persistence/media_queries.go b/pkg/persistence/media_queries.go index dec793b..563624a 100644 --- a/pkg/persistence/media_queries.go +++ b/pkg/persistence/media_queries.go @@ -3,14 +3,14 @@ package persistence import ( "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Save an Image // // args: // - img: the Image to save -func (p Profile) SaveImage(img scraper.Image) error { +func (p Profile) SaveImage(img Image) error { _, err := p.DB.NamedExec(` insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded) values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :is_downloaded) @@ -29,7 +29,7 @@ func (p Profile) SaveImage(img scraper.Image) error { // // args: // - img: the Video to save -func (p Profile) SaveVideo(vid scraper.Video) error { +func (p Profile) SaveVideo(vid Video) error { _, err := p.DB.NamedExec(` insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif) @@ -49,7 +49,7 @@ func (p Profile) SaveVideo(vid scraper.Video) error { } // Save an Url -func (p Profile) SaveUrl(url scraper.Url) error { +func (p Profile) SaveUrl(url Url) error { _, err := p.DB.NamedExec(` insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) @@ -68,7 +68,7 @@ func (p Profile) SaveUrl(url scraper.Url) error { } // Save a Poll -func (p Profile) SavePoll(poll scraper.Poll) error { +func (p Profile) SavePoll(poll Poll) error { _, err := p.DB.NamedExec(` insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, voting_duration, voting_ends_at, last_scraped_at) @@ -90,7 +90,7 @@ func (p Profile) SavePoll(poll scraper.Poll) error { } // Get the list of images for a tweet -func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) { +func (p Profile) GetImagesForTweet(t Tweet) (imgs []Image, err error) { err = p.DB.Select(&imgs, "select id, tweet_id, width, height, remote_url, local_filename, is_downloaded from images where tweet_id=?", t.ID) @@ -98,7 +98,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e } // Get the list of videos for a tweet -func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) { +func (p Profile) GetVideosForTweet(t Tweet) (vids []Video, err error) { err = p.DB.Select(&vids, ` select id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif @@ -109,7 +109,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e } // Get the list of Urls for a Tweet -func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error) { +func (p Profile) GetUrlsForTweet(t Tweet) (urls []Url, err error) { err = p.DB.Select(&urls, ` select tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded @@ -121,7 +121,7 @@ func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error } // Get the list of Polls for a Tweet -func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err error) { +func (p Profile) GetPollsForTweet(t Tweet) (polls []Poll, err error) { err = p.DB.Select(&polls, ` select id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, voting_duration, voting_ends_at, last_scraped_at diff --git a/pkg/persistence/media_queries_test.go b/pkg/persistence/media_queries_test.go index 39967e0..cd826e1 100644 --- a/pkg/persistence/media_queries_test.go +++ b/pkg/persistence/media_queries_test.go @@ -8,7 +8,7 @@ import ( "github.com/go-test/deep" "github.com/stretchr/testify/require" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create an Image, save it, reload it, and make sure it comes back the same @@ -31,7 +31,7 @@ func TestSaveAndLoadImage(t *testing.T) { imgs, err := profile.GetImagesForTweet(tweet) require.NoError(err) - var new_img scraper.Image + var new_img Image for index := range imgs { if imgs[index].ID == img.ID { new_img = imgs[index] @@ -52,7 +52,7 @@ func TestModifyImage(t *testing.T) { tweet := create_stable_tweet() img := tweet.Images[0] - require.Equal(scraper.ImageID(-1), img.ID, "Got the wrong image back") + require.Equal(ImageID(-1), img.ID, "Got the wrong image back") img.IsDownloaded = true @@ -94,7 +94,7 @@ func TestSaveAndLoadVideo(t *testing.T) { vids, err := profile.GetVideosForTweet(tweet) require.NoError(err) - var new_vid scraper.Video + var new_vid Video for index := range vids { if vids[index].ID == vid.ID { new_vid = vids[index] @@ -115,7 +115,7 @@ func TestModifyVideo(t *testing.T) { tweet := create_stable_tweet() vid := tweet.Videos[0] - require.Equal(scraper.VideoID(-1), vid.ID, "Got the wrong video back") + require.Equal(VideoID(-1), vid.ID, "Got the wrong video back") vid.IsDownloaded = true vid.IsBlockedByDMCA = true @@ -157,7 +157,7 @@ func TestSaveAndLoadUrl(t *testing.T) { urls, err := profile.GetUrlsForTweet(tweet) require.NoError(err) - var new_url scraper.Url + var new_url Url for index := range urls { if urls[index].Text == url.Text { new_url = urls[index] @@ -218,7 +218,7 @@ func TestSaveAndLoadPoll(t *testing.T) { polls, err := profile.GetPollsForTweet(tweet) require.NoError(err) - var new_poll scraper.Poll + var new_poll Poll for index := range polls { if polls[index].ID == poll.ID { new_poll = polls[index] diff --git a/pkg/persistence/notification_queries.go b/pkg/persistence/notification_queries.go index 36adb76..08aaeae 100644 --- a/pkg/persistence/notification_queries.go +++ b/pkg/persistence/notification_queries.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/persistence/profile.go b/pkg/persistence/profile.go index 0648a27..dc0eb5e 100644 --- a/pkg/persistence/profile.go +++ b/pkg/persistence/profile.go @@ -9,7 +9,7 @@ import ( sql "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) //go:embed schema.sql @@ -125,14 +125,14 @@ func LoadProfile(profile_dir string) (Profile, error) { return ret, err } -func (p Profile) ListSessions() []scraper.UserHandle { +func (p Profile) ListSessions() []UserHandle { result, err := filepath.Glob(filepath.Join(p.ProfileDir, "*.session")) if err != nil { panic(err) } - ret := []scraper.UserHandle{} + ret := []UserHandle{} for _, filename := range result { - ret = append(ret, scraper.UserHandle(filepath.Base(filename[:len(filename)-len(".session")]))) + ret = append(ret, UserHandle(filepath.Base(filename[:len(filename)-len(".session")]))) } return ret } diff --git a/pkg/persistence/retweet_queries.go b/pkg/persistence/retweet_queries.go index b035688..a953be6 100644 --- a/pkg/persistence/retweet_queries.go +++ b/pkg/persistence/retweet_queries.go @@ -3,11 +3,11 @@ package persistence import ( "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable. -func (p Profile) SaveRetweet(r scraper.Retweet) error { +func (p Profile) SaveRetweet(r Retweet) error { _, err := p.DB.NamedExec(` insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at) values (:retweet_id, :tweet_id, :retweeted_by, :retweeted_at) @@ -22,8 +22,8 @@ func (p Profile) SaveRetweet(r scraper.Retweet) error { } // Retrieve a Retweet by ID -func (p Profile) GetRetweetById(id scraper.TweetID) (scraper.Retweet, error) { - var r scraper.Retweet +func (p Profile) GetRetweetById(id TweetID) (Retweet, error) { + var r Retweet err := p.DB.Get(&r, ` select retweet_id, tweet_id, retweeted_by, retweeted_at from retweets diff --git a/pkg/persistence/session.go b/pkg/persistence/session.go index d659c16..aa6759c 100644 --- a/pkg/persistence/session.go +++ b/pkg/persistence/session.go @@ -2,13 +2,15 @@ package persistence import ( "encoding/json" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + "os" log "github.com/sirupsen/logrus" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) -func (p Profile) SaveSession(api scraper.API) { +func (p Profile) SaveSession(api API) { data, err := json.Marshal(api) if err != nil { panic(err) @@ -21,13 +23,13 @@ func (p Profile) SaveSession(api scraper.API) { } } -func (p Profile) LoadSession(userhandle scraper.UserHandle) scraper.API { +func (p Profile) LoadSession(userhandle UserHandle) API { data, err := os.ReadFile(p.ProfileDir + "/" + string(userhandle+".session")) if err != nil { panic(err) } - var result scraper.API + var result API err = json.Unmarshal(data, &result) if err != nil { panic(err) diff --git a/pkg/persistence/session_test.go b/pkg/persistence/session_test.go index 2011b1a..c4ef069 100644 --- a/pkg/persistence/session_test.go +++ b/pkg/persistence/session_test.go @@ -2,7 +2,7 @@ package persistence_test import ( "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + "math/rand" "net/http" "net/http/cookiejar" @@ -10,6 +10,8 @@ import ( "time" "github.com/go-test/deep" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Save and load an API session; it should come back the same @@ -22,7 +24,7 @@ func TestSaveAndLoadAuthenticatedSession(t *testing.T) { panic(err) } - api := scraper.API{ + api := API{ UserHandle: "testUser", IsAuthenticated: true, Client: http.Client{ diff --git a/pkg/persistence/space_queries.go b/pkg/persistence/space_queries.go index 551fbcf..672a2d1 100644 --- a/pkg/persistence/space_queries.go +++ b/pkg/persistence/space_queries.go @@ -4,16 +4,17 @@ import ( "database/sql" "errors" "fmt" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type SpaceParticipant struct { - UserID scraper.UserID `db:"user_id"` - SpaceID scraper.SpaceID `db:"space_id"` + UserID UserID `db:"user_id"` + SpaceID SpaceID `db:"space_id"` } // Save a Space -func (p Profile) SaveSpace(s scraper.Space) error { +func (p Profile) SaveSpace(s Space) error { _, err := p.DB.NamedExec(` insert into spaces (id, created_by_id, short_url, state, title, created_at, started_at, ended_at, updated_at, is_available_for_replay, replay_watch_count, live_listeners_count, is_details_fetched) @@ -52,7 +53,7 @@ func (p Profile) SaveSpace(s scraper.Space) error { } // Get a Space by ID -func (p Profile) GetSpaceById(id scraper.SpaceID) (space scraper.Space, err error) { +func (p Profile) GetSpaceById(id SpaceID) (space Space, err error) { err = p.DB.Get(&space, `select id, ifnull(created_by_id, 0) created_by_id, short_url, state, title, created_at, started_at, ended_at, updated_at, is_available_for_replay, replay_watch_count, live_listeners_count, is_details_fetched @@ -61,7 +62,7 @@ func (p Profile) GetSpaceById(id scraper.SpaceID) (space scraper.Space, err erro if err != nil { return } - space.ParticipantIds = []scraper.UserID{} + space.ParticipantIds = []UserID{} rows, err := p.DB.Query(`select user_id from space_participants where space_id = ?`, id) if errors.Is(err, sql.ErrNoRows) { err = nil @@ -70,7 +71,7 @@ func (p Profile) GetSpaceById(id scraper.SpaceID) (space scraper.Space, err erro if err != nil { panic(err) } - var participant_id scraper.UserID + var participant_id UserID for rows.Next() { err = rows.Scan(&participant_id) if err != nil { diff --git a/pkg/persistence/space_queries_test.go b/pkg/persistence/space_queries_test.go index eea80f9..06323c2 100644 --- a/pkg/persistence/space_queries_test.go +++ b/pkg/persistence/space_queries_test.go @@ -5,10 +5,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" "github.com/go-test/deep" "math/rand" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create a Space, save it, reload it, and make sure it comes back the same @@ -41,8 +42,8 @@ func TestModifySpace(t *testing.T) { // Modify and save space.State = "Some other state" - space.UpdatedAt = scraper.TimestampFromUnix(9001) - space.EndedAt = scraper.TimestampFromUnix(10001) + space.UpdatedAt = TimestampFromUnix(9001) + space.EndedAt = TimestampFromUnix(10001) space.ReplayWatchCount = 100 space.LiveListenersCount = 50 space.IsDetailsFetched = true @@ -51,8 +52,8 @@ func TestModifySpace(t *testing.T) { new_space, err := profile.GetSpaceById(space.ID) require.NoError(err) - assert.Equal(scraper.TimestampFromUnix(9001), new_space.UpdatedAt) - assert.Equal(scraper.TimestampFromUnix(10001), new_space.EndedAt) + assert.Equal(TimestampFromUnix(9001), new_space.UpdatedAt) + assert.Equal(TimestampFromUnix(10001), new_space.EndedAt) assert.Equal(100, new_space.ReplayWatchCount) assert.Equal(50, new_space.LiveListenersCount) assert.True(new_space.IsDetailsFetched) @@ -68,9 +69,9 @@ func TestNoWorseningSpace(t *testing.T) { space.ShortUrl = "Some Short Url" space.State = "Some State" space.Title = "Debating Somebody" - space.CreatedAt = scraper.TimestampFromUnix(1000) - space.UpdatedAt = scraper.TimestampFromUnix(2000) - space.CreatedById = scraper.UserID(-1) + space.CreatedAt = TimestampFromUnix(1000) + space.UpdatedAt = TimestampFromUnix(2000) + space.CreatedById = UserID(-1) space.LiveListenersCount = 100 space.IsDetailsFetched = true @@ -82,9 +83,9 @@ func TestNoWorseningSpace(t *testing.T) { space.ShortUrl = "" space.Title = "" space.State = "" - space.CreatedAt = scraper.TimestampFromUnix(0) - space.UpdatedAt = scraper.TimestampFromUnix(0) - space.CreatedById = scraper.UserID(0) + space.CreatedAt = TimestampFromUnix(0) + space.UpdatedAt = TimestampFromUnix(0) + space.CreatedById = UserID(0) space.LiveListenersCount = 0 space.IsDetailsFetched = false err = profile.SaveSpace(space) @@ -97,9 +98,9 @@ func TestNoWorseningSpace(t *testing.T) { assert.Equal(new_space.ShortUrl, "Some Short Url") assert.Equal(new_space.State, "Some State") assert.Equal(new_space.Title, "Debating Somebody") - assert.Equal(new_space.CreatedAt, scraper.TimestampFromUnix(1000)) - assert.Equal(new_space.UpdatedAt, scraper.TimestampFromUnix(2000)) - assert.Equal(new_space.CreatedById, scraper.UserID(-1)) + assert.Equal(new_space.CreatedAt, TimestampFromUnix(1000)) + assert.Equal(new_space.UpdatedAt, TimestampFromUnix(2000)) + assert.Equal(new_space.CreatedById, UserID(-1)) assert.Equal(new_space.LiveListenersCount, 100) assert.True(new_space.IsDetailsFetched) } diff --git a/pkg/persistence/tweet_queries_test.go b/pkg/persistence/tweet_queries_test.go index a48e865..05bcec1 100644 --- a/pkg/persistence/tweet_queries_test.go +++ b/pkg/persistence/tweet_queries_test.go @@ -9,7 +9,7 @@ import ( "github.com/go-test/deep" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create a Tweet, save it, reload it, and make sure it comes back the same @@ -71,7 +71,7 @@ func TestNoWorseningTweet(t *testing.T) { tweet.IsStub = false tweet.IsConversationScraped = true tweet.IsExpandable = true - tweet.LastScrapedAt = scraper.TimestampFromUnix(1000) + tweet.LastScrapedAt = TimestampFromUnix(1000) tweet.Text = "Yes text" tweet.NumLikes = 10 tweet.NumRetweets = 11 @@ -87,7 +87,7 @@ func TestNoWorseningTweet(t *testing.T) { tweet.IsStub = true tweet.IsConversationScraped = false tweet.IsExpandable = false - tweet.LastScrapedAt = scraper.TimestampFromUnix(500) + tweet.LastScrapedAt = TimestampFromUnix(500) tweet.Text = "" err = profile.SaveTweet(tweet) require.NoError(err) @@ -231,7 +231,7 @@ func TestModifyTweet(t *testing.T) { tweet.IsStub = true tweet.IsContentDownloaded = false tweet.IsConversationScraped = false - tweet.LastScrapedAt = scraper.TimestampFromUnix(1000) + tweet.LastScrapedAt = TimestampFromUnix(1000) err := profile.SaveTweet(tweet) require.NoError(err) @@ -243,7 +243,7 @@ func TestModifyTweet(t *testing.T) { tweet.IsStub = false tweet.IsContentDownloaded = true tweet.IsConversationScraped = true - tweet.LastScrapedAt = scraper.TimestampFromUnix(2000) + tweet.LastScrapedAt = TimestampFromUnix(2000) tweet.TombstoneType = "deleted" err = profile.SaveTweet(tweet) @@ -332,7 +332,7 @@ func TestLoadMissingTweet(t *testing.T) { profile_path := "test_profiles/TestTweetQueries" profile := create_or_load_profile(profile_path) - _, err := profile.GetTweetById(scraper.TweetID(6234234)) // Random number + _, err := profile.GetTweetById(TweetID(6234234)) // Random number require.Error(t, err) assert.ErrorIs(t, err, persistence.ErrNotInDatabase) } diff --git a/pkg/persistence/tweet_trove_queries.go b/pkg/persistence/tweet_trove_queries.go index c39806b..7407e9a 100644 --- a/pkg/persistence/tweet_trove_queries.go +++ b/pkg/persistence/tweet_trove_queries.go @@ -5,6 +5,8 @@ import ( "fmt" "path" + log "github.com/sirupsen/logrus" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -18,7 +20,7 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API // who were marked as deleted, and then let the callee re-scrape and re-save them. var conflict_err ErrConflictingUserHandle if errors.As(err, &conflict_err) { - fmt.Printf( + log.Warnf( "Conflicting user handle found (ID %d); old user has been marked deleted. Rescraping them\n", conflict_err.ConflictingUserID, ) diff --git a/pkg/persistence/user_queries.go b/pkg/persistence/user_queries.go index dace0b0..3485bef 100644 --- a/pkg/persistence/user_queries.go +++ b/pkg/persistence/user_queries.go @@ -9,11 +9,11 @@ import ( "github.com/jmoiron/sqlx" "github.com/mattn/go-sqlite3" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type ErrConflictingUserHandle struct { - ConflictingUserID scraper.UserID + ConflictingUserID UserID } func (e ErrConflictingUserHandle) Error() string { @@ -46,7 +46,7 @@ const USERS_ALL_SQL_FIELDS = ` // 3. Mark the old user as deactivated, eliminating the conflict // 4. Re-save the new user // 5. Return an ErrConflictingUserHandle, notifying the caller of the conflict -func (p Profile) SaveUser(u *scraper.User) error { +func (p Profile) SaveUser(u *User) error { // First, check if the user needs a fake ID, and generate one if needed if u.IsNeedingFakeID { // User is fake; check if we already have them, in order to proceed @@ -70,7 +70,7 @@ func (p Profile) SaveUser(u *scraper.User) error { // We know the UNIQUE violation must be on `handle`, because we checked for users with this ID // above (`update` query). handle_conflict := func() error { - var old_user scraper.User + var old_user User err := p.DB.Get(&old_user, `select id, is_id_fake from users where handle = ? and is_banned = 0 and is_deleted = 0`, u.Handle, @@ -189,10 +189,10 @@ func (p Profile) SaveUser(u *scraper.User) error { // // returns: // - the User, if it exists -func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error) { +func (p Profile) GetUserByHandle(handle UserHandle) (User, error) { db := p.DB - var ret scraper.User + var ret User err := db.Get(&ret, ` select `+USERS_ALL_SQL_FIELDS+` from users_by_handle @@ -212,10 +212,10 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error // // returns: // - the User, if it exists -func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) { +func (p Profile) GetUserByID(id UserID) (User, error) { db := p.DB - var ret scraper.User + var ret User err := db.Get(&ret, ` select `+USERS_ALL_SQL_FIELDS+` @@ -243,7 +243,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) { // // The `user` object will always have `is_content_downloaded` = false on every scrape. This is // why the No Worsening Principle is needed. -func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool { +func (p Profile) CheckUserContentDownloadNeeded(user User) bool { row := p.DB.QueryRow(`select is_content_downloaded, profile_image_url, banner_image_url from users where id = ?`, user.ID) var is_content_downloaded bool @@ -271,7 +271,7 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool { } // Follow / unfollow a user. Update the given User object's IsFollowed field. -func (p Profile) SetUserFollowed(user *scraper.User, is_followed bool) { +func (p Profile) SetUserFollowed(user *User, is_followed bool) { result, err := p.DB.Exec("update users set is_followed = ? where id = ?", is_followed, user.ID) if err != nil { panic(fmt.Errorf("Error inserting user with handle %q:\n %w", user.Handle, err)) @@ -286,12 +286,12 @@ func (p Profile) SetUserFollowed(user *scraper.User, is_followed bool) { user.IsFollowed = is_followed } -func (p Profile) NextFakeUserID() scraper.UserID { +func (p Profile) NextFakeUserID() UserID { _, err := p.DB.Exec("update fake_user_sequence set latest_fake_id = latest_fake_id + 1") if err != nil { panic(err) } - var ret scraper.UserID + var ret UserID err = p.DB.QueryRow("select latest_fake_id from fake_user_sequence").Scan(&ret) if err != nil { panic(err) @@ -301,7 +301,7 @@ func (p Profile) NextFakeUserID() scraper.UserID { // TODO: This is only used in checking whether the media downloader should get the big or small version of // a profile image. That should be rewritten -func (p Profile) IsFollowing(user scraper.User) bool { +func (p Profile) IsFollowing(user User) bool { row := p.DB.QueryRow("select is_followed from users where id like ?", user.ID) var ret bool err := row.Scan(&ret) @@ -315,21 +315,21 @@ func (p Profile) IsFollowing(user scraper.User) bool { } // Utility function to compute the path to save banner image to -func (p Profile) get_banner_image_output_path(u scraper.User) string { +func (p Profile) get_banner_image_output_path(u User) string { return path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath) } // Utility function to compute the path to save profile image to -func (p Profile) get_profile_image_output_path(u scraper.User) string { +func (p Profile) get_profile_image_output_path(u User) string { if u.ProfileImageUrl == "" { - return path.Join(p.ProfileDir, "profile_images", path.Base(scraper.DEFAULT_PROFILE_IMAGE_URL)) + return path.Join(p.ProfileDir, "profile_images", path.Base(DEFAULT_PROFILE_IMAGE_URL)) } return path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath) } // Do a text search for users -func (p Profile) SearchUsers(s string) []scraper.User { - var ret []scraper.User +func (p Profile) SearchUsers(s string) []User { + var ret []User q, args, err := sqlx.Named(` select `+USERS_ALL_SQL_FIELDS+` from users diff --git a/pkg/persistence/user_queries_test.go b/pkg/persistence/user_queries_test.go index 46ac05c..140fc5f 100644 --- a/pkg/persistence/user_queries_test.go +++ b/pkg/persistence/user_queries_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create a user, save it, reload it, and make sure it comes back the same @@ -53,19 +53,19 @@ func TestModifyUser(t *testing.T) { user := create_dummy_user() user.DisplayName = "Display Name 1" user.Location = "location1" - user.Handle = scraper.UserHandle(fmt.Sprintf("handle %d", rand.Int31())) + user.Handle = UserHandle(fmt.Sprintf("handle %d", rand.Int31())) user.IsPrivate = false user.IsVerified = false user.FollowersCount = 1000 - user.JoinDate = scraper.TimestampFromUnix(1000) + user.JoinDate = TimestampFromUnix(1000) user.ProfileImageUrl = "asdf" user.IsContentDownloaded = true - // Save the user so it can be modified + // Save the user for the first time; should do insert err := profile.SaveUser(&user) require.NoError(err) - new_handle := scraper.UserHandle(fmt.Sprintf("handle %d", rand.Int31())) + new_handle := UserHandle(fmt.Sprintf("handle %d", rand.Int31())) user.DisplayName = "Display Name 2" user.Location = "location2" @@ -73,11 +73,11 @@ func TestModifyUser(t *testing.T) { user.IsPrivate = true user.IsVerified = true user.FollowersCount = 2000 - user.JoinDate = scraper.TimestampFromUnix(2000) + user.JoinDate = TimestampFromUnix(2000) user.ProfileImageUrl = "asdf2" user.IsContentDownloaded = false // test No Worsening - // Save the modified user + // Save the user for the second time; should do update err = profile.SaveUser(&user) require.NoError(err) @@ -107,9 +107,9 @@ func TestSetUserBannedDeleted(t *testing.T) { user.DisplayName = "Display Name 1" user.Location = "location1" user.Bio = "Some Bio" - user.Handle = scraper.UserHandle(fmt.Sprintf("handle %d", rand.Int31())) + user.Handle = UserHandle(fmt.Sprintf("handle %d", rand.Int31())) user.FollowersCount = 1000 - user.JoinDate = scraper.TimestampFromUnix(1000) + user.JoinDate = TimestampFromUnix(1000) user.ProfileImageUrl = "asdf" user.IsContentDownloaded = true @@ -118,7 +118,7 @@ func TestSetUserBannedDeleted(t *testing.T) { require.NoError(err) // Now the user deactivates - err = profile.SaveUser(&scraper.User{ID: user.ID, IsDeleted: true}) + err = profile.SaveUser(&User{ID: user.ID, IsDeleted: true}) require.NoError(err) // Reload the modified user new_user, err := profile.GetUserByID(user.ID) @@ -141,9 +141,9 @@ func TestSaveAndLoadBannedDeletedUser(t *testing.T) { profile_path := "test_profiles/TestUserQueries" profile := create_or_load_profile(profile_path) - user := scraper.User{ - ID: scraper.UserID(rand.Int31()), - Handle: scraper.UserHandle(fmt.Sprintf("handle-%d", rand.Int31())), + user := User{ + ID: UserID(rand.Int31()), + Handle: UserHandle(fmt.Sprintf("handle-%d", rand.Int31())), IsBanned: true, } @@ -365,20 +365,20 @@ func TestCreateUnknownUserWithHandle(t *testing.T) { next_id := profile.NextFakeUserID() - handle := scraper.UserHandle(fmt.Sprintf("UnknownUser%d", rand.Int31())) - user := scraper.GetUnknownUserWithHandle(handle) - assert.Equal(scraper.UserID(0), user.ID) + handle := UserHandle(fmt.Sprintf("UnknownUser%d", rand.Int31())) + user := GetUnknownUserWithHandle(handle) + assert.Equal(UserID(0), user.ID) assert.True(user.IsIdFake) err := profile.SaveUser(&user) assert.NoError(err) - assert.Equal(scraper.UserID(next_id+1), user.ID) + assert.Equal(UserID(next_id+1), user.ID) // Ensure the change was persisted user_reloaded, err := profile.GetUserByHandle(user.Handle) require.NoError(t, err) assert.Equal(handle, user_reloaded.Handle) // Verify it's the same user - assert.Equal(scraper.UserID(next_id+1), user_reloaded.ID) + assert.Equal(UserID(next_id+1), user_reloaded.ID) // Why not tack this test on here: make sure NextFakeUserID works as expected assert.Equal(next_id+2, profile.NextFakeUserID()) @@ -393,8 +393,8 @@ func TestCreateUnknownUserWithHandleThatAlreadyExists(t *testing.T) { user := create_stable_user() - unknown_user := scraper.GetUnknownUserWithHandle(user.Handle) - assert.Equal(scraper.UserID(0), unknown_user.ID) + unknown_user := GetUnknownUserWithHandle(user.Handle) + assert.Equal(UserID(0), unknown_user.ID) err := profile.SaveUser(&unknown_user) assert.NoError(err) @@ -417,6 +417,6 @@ func TestSearchUsers(t *testing.T) { users := profile.SearchUsers("no") assert.Len(users, 2) - assert.Equal(users[0].Handle, scraper.UserHandle("Cernovich")) - assert.Equal(users[1].Handle, scraper.UserHandle("CovfefeAnon")) + assert.Equal(users[0].Handle, UserHandle("Cernovich")) + assert.Equal(users[1].Handle, UserHandle("CovfefeAnon")) } diff --git a/pkg/persistence/versions_test.go b/pkg/persistence/versions_test.go index 7003494..18909a1 100644 --- a/pkg/persistence/versions_test.go +++ b/pkg/persistence/versions_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestVersionUpgrade(t *testing.T) { @@ -21,7 +21,7 @@ func TestVersionUpgrade(t *testing.T) { profile := create_or_load_profile(profile_path) test_migration := "insert into tweets (id, user_id, text) values (21250554358298342, -1, 'awefjk')" - test_tweet_id := scraper.TweetID(21250554358298342) + test_tweet_id := TweetID(21250554358298342) require.False(profile.IsTweetInDatabase(test_tweet_id), "Test tweet shouldn't be in db yet")