Add downloading of User banner image and profile image

persistence/media_download.go

@@ -102,3 +102,36 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M
     t.IsContentDownloaded = true
     return p.SaveTweet(*t)
 }
+
+/**
+ * Download a user's banner and profile images
+ */
+func (p Profile) DownloadUserContentFor(u *scraper.User) error {
+    return p.DownloadUserContentWithInjector(u, DefaultDownloader{})
+}
+
+/**
+ * Enable injecting a custom MediaDownloader (i.e., for testing)
+ */
+func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error {
+    var err error
+    var outfile string
+
+    outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath)
+    err = downloader.Curl(u.ProfileImageUrl, outfile)
+    if err != nil {
+        return err
+    }
+
+    // Skip it if there's no banner image
+    if u.BannerImageLocalPath != "" {
+        outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath)
+        err = downloader.Curl(u.BannerImageUrl, outfile)
+        if err != nil {
+            return err
+        }
+    }
+
+    u.IsContentDownloaded = true
+    return p.SaveUser(*u)
+}

persistence/media_download_test.go

@@ -37,7 +37,7 @@ func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) {
 }
 
 /**
- * Create an Image, save it, reload it, and make sure it comes back the same
+ * Downloading a Tweet's contents should mark the Tweet as downloaded
  */
 func TestDownloadTweetContent(t *testing.T) {
     profile_path := "test_profiles/TestMediaQueries"
@@ -63,10 +63,50 @@ func TestDownloadTweetContent(t *testing.T) {
     // It should all be marked "yes downloaded" now
     test_all_downloaded(tweet, true, t)
 
-    // Reload the tweet (check db); should also be "yes downloaded"
+    // Reload the Tweet (check db); should also be "yes downloaded"
     new_tweet, err := profile.GetTweetById(tweet.ID)
     if err != nil {
-        t.Fatalf("Couldn't reload the tweet: %s", err.Error())
+        t.Fatalf("Couldn't reload the Tweet: %s", err.Error())
     }
     test_all_downloaded(new_tweet, true, t)
 }
+
+/**
+ * Downloading a User's contents should mark the User as downloaded
+ */
+func TestDownloadUserContent(t *testing.T) {
+    profile_path := "test_profiles/TestMediaQueries"
+    profile := create_or_load_profile(profile_path)
+
+    user := create_dummy_user()
+
+    // Persist the User
+    err := profile.SaveUser(user)
+    if err != nil {
+        t.Fatalf("Failed to save the user: %s", err.Error())
+    }
+
+    // Make sure the User is marked "not downloaded"
+    if user.IsContentDownloaded {
+        t.Errorf("User shouldn't be marked downloaded, but it was")
+    }
+
+    // Do the (fake) downloading
+    err = profile.DownloadUserContentWithInjector(&user, FakeDownloader{})
+    if err != nil {
+        t.Fatalf("Error running fake download: %s", err.Error())
+    }
+
+    // The User should now be marked "yes downloaded"
+    if !user.IsContentDownloaded {
+        t.Errorf("User should be marked downloaded, but it wasn't")
+    }
+
+    // Reload the User (check db); should also be "yes downloaded"
+    new_user, err := profile.GetUserByID(user.ID)
+    if err != nil {
+        t.Fatalf("Couldn't reload the User: %s", err.Error())
+    }
+    if !new_user.IsContentDownloaded {
+        t.Errorf("User should be marked downloaded, but it wasn't")
+    }}

persistence/user_queries.go

@@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error {
         return err
     }
     _, err = db.Exec(`
-        insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id)
+        insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded)
-        values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             on conflict do update
            set bio=?,
                   following_count=?,
@@ -35,10 +35,11 @@ func (p Profile) SaveUser(u scraper.User) error {
                   profile_image_local_path=?,
                   banner_image_url=?,
                   banner_image_local_path=?,
-                  pinned_tweet_id=?
+                  pinned_tweet_id=?,
+                  is_content_downloaded=?
         `,
-        u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
+        u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
-        u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
+        u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
     )
     if err != nil {
         return err
@@ -84,7 +85,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) {
     var u scraper.User
     var joinDate int64
 
-    err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID)
+    err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID, &u.IsContentDownloaded)
     if err != nil {
         return u, err
     }
@@ -107,7 +108,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
     db := p.DB
 
     stmt, err := db.Prepare(`
-        select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
+        select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded
           from users
          where handle = ?
     `)
@@ -138,7 +139,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
     db := p.DB
 
     stmt, err := db.Prepare(`
-        select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
+        select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded
           from users
          where id = ?
     `)