From 3d289ed5f0f4f0c5b0de2c1c37f25b415b636308 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sat, 7 May 2022 18:46:07 -0700 Subject: [PATCH] Profile images are now downloaded based on whether the local file path exists, not just what the DB says --- persistence/tweet_trove_queries.go | 14 +++++++------- persistence/user_queries.go | 7 +++++-- persistence/user_queries_test.go | 11 +++++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/persistence/tweet_trove_queries.go b/persistence/tweet_trove_queries.go index e5adeea..9c51773 100644 --- a/persistence/tweet_trove_queries.go +++ b/persistence/tweet_trove_queries.go @@ -12,19 +12,19 @@ import ( */ func (p Profile) SaveTweetTrove(trove TweetTrove) { for i, u := range trove.Users { - // Download download their tiny profile image - err := p.DownloadUserProfileImageTiny(&u) - if err != nil { - panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err)) - } - - err = p.SaveUser(&u) + err := p.SaveUser(&u) if err != nil { panic(fmt.Errorf("Error saving user with ID %d and handle %s:\n %w", u.ID, u.Handle, err)) } fmt.Println(u.Handle, u.ID) // If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too trove.Users[i] = u + + // Download their tiny profile image + err = p.DownloadUserProfileImageTiny(&u) + if err != nil { + panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err)) + } } // TODO: this is called earlier in the process as well, before parsing. Is that call redundant? Too tired to figure out right now diff --git a/persistence/user_queries.go b/persistence/user_queries.go index e6282f7..f94ddf6 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -181,10 +181,13 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool { if !is_content_downloaded { return true } - if banner_image_url != user.BannerImageUrl { + + banner_path := p.get_banner_image_output_path(user) + if banner_path != "" && !file_exists(banner_path) { return true } - if profile_image_url != user.ProfileImageUrl { + profile_path := p.get_profile_image_output_path(user) + if !file_exists(profile_path) { return true } return false diff --git a/persistence/user_queries_test.go b/persistence/user_queries_test.go index 95f3afb..e15ca46 100644 --- a/persistence/user_queries_test.go +++ b/persistence/user_queries_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" + "os" "github.com/go-test/deep" "github.com/stretchr/testify/assert" @@ -164,13 +165,15 @@ func TestCheckUserContentDownloadNeeded(t *testing.T) { err = profile.SaveUser(&user) require.NoError(t, err) - // If everything is up to date, no download should be required + // If the file exists, then it should not require any download + _, err = os.Create("test_profiles/TestUserQueries/profile_images/" + user.BannerImageLocalPath) + require.NoError(t, err) + _, err = os.Create("test_profiles/TestUserQueries/profile_images/" + user.ProfileImageLocalPath) + require.NoError(t, err) assert.False(profile.CheckUserContentDownloadNeeded(user)) - // Change an URL, but don't save it-- needs to be different from what's in the DB - user.BannerImageUrl = "banner url2" - // Download needed for new banner image + user.BannerImageLocalPath = "banner url2" assert.True(profile.CheckUserContentDownloadNeeded(user)) }