From 294917124a465a5e5067e5d073dacb5fdd58228c Mon Sep 17 00:00:00 2001 From: Alessio Date: Tue, 10 Aug 2021 21:35:27 -0700 Subject: [PATCH] Add downloading of User banner image and profile image --- persistence/media_download.go | 33 +++++++++++++++++++++ persistence/media_download_test.go | 46 ++++++++++++++++++++++++++++-- persistence/user_queries.go | 17 +++++------ 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/persistence/media_download.go b/persistence/media_download.go index bfc7ed4..db2325d 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -102,3 +102,36 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M t.IsContentDownloaded = true return p.SaveTweet(*t) } + +/** + * Download a user's banner and profile images + */ +func (p Profile) DownloadUserContentFor(u *scraper.User) error { + return p.DownloadUserContentWithInjector(u, DefaultDownloader{}) +} + +/** + * Enable injecting a custom MediaDownloader (i.e., for testing) + */ +func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error { + var err error + var outfile string + + outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath) + err = downloader.Curl(u.ProfileImageUrl, outfile) + if err != nil { + return err + } + + // Skip it if there's no banner image + if u.BannerImageLocalPath != "" { + outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath) + err = downloader.Curl(u.BannerImageUrl, outfile) + if err != nil { + return err + } + } + + u.IsContentDownloaded = true + return p.SaveUser(*u) +} diff --git a/persistence/media_download_test.go b/persistence/media_download_test.go index 267a0e1..6ed7624 100644 --- a/persistence/media_download_test.go +++ b/persistence/media_download_test.go @@ -37,7 +37,7 @@ func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) { } /** - * Create an Image, save it, reload it, and make sure it comes back the same + * Downloading a Tweet's contents should mark the Tweet as downloaded */ func TestDownloadTweetContent(t *testing.T) { profile_path := "test_profiles/TestMediaQueries" @@ -63,10 +63,50 @@ func TestDownloadTweetContent(t *testing.T) { // It should all be marked "yes downloaded" now test_all_downloaded(tweet, true, t) - // Reload the tweet (check db); should also be "yes downloaded" + // Reload the Tweet (check db); should also be "yes downloaded" new_tweet, err := profile.GetTweetById(tweet.ID) if err != nil { - t.Fatalf("Couldn't reload the tweet: %s", err.Error()) + t.Fatalf("Couldn't reload the Tweet: %s", err.Error()) } test_all_downloaded(new_tweet, true, t) } + +/** + * Downloading a User's contents should mark the User as downloaded + */ +func TestDownloadUserContent(t *testing.T) { + profile_path := "test_profiles/TestMediaQueries" + profile := create_or_load_profile(profile_path) + + user := create_dummy_user() + + // Persist the User + err := profile.SaveUser(user) + if err != nil { + t.Fatalf("Failed to save the user: %s", err.Error()) + } + + // Make sure the User is marked "not downloaded" + if user.IsContentDownloaded { + t.Errorf("User shouldn't be marked downloaded, but it was") + } + + // Do the (fake) downloading + err = profile.DownloadUserContentWithInjector(&user, FakeDownloader{}) + if err != nil { + t.Fatalf("Error running fake download: %s", err.Error()) + } + + // The User should now be marked "yes downloaded" + if !user.IsContentDownloaded { + t.Errorf("User should be marked downloaded, but it wasn't") + } + + // Reload the User (check db); should also be "yes downloaded" + new_user, err := profile.GetUserByID(user.ID) + if err != nil { + t.Fatalf("Couldn't reload the User: %s", err.Error()) + } + if !new_user.IsContentDownloaded { + t.Errorf("User should be marked downloaded, but it wasn't") + }} diff --git a/persistence/user_queries.go b/persistence/user_queries.go index 26439d8..f4a54b7 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error { return err } _, err = db.Exec(` - insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set bio=?, following_count=?, @@ -35,10 +35,11 @@ func (p Profile) SaveUser(u scraper.User) error { profile_image_local_path=?, banner_image_url=?, banner_image_local_path=?, - pinned_tweet_id=? + pinned_tweet_id=?, + is_content_downloaded=? `, - u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, - u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, + u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, + u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, ) if err != nil { return err @@ -84,7 +85,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) { var u scraper.User var joinDate int64 - err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID) + err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID, &u.IsContentDownloaded) if err != nil { return u, err } @@ -107,7 +108,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error db := p.DB stmt, err := db.Prepare(` - select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id + select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded from users where handle = ? `) @@ -138,7 +139,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) { db := p.DB stmt, err := db.Prepare(` - select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id + select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded from users where id = ? `)