me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Profile images are now downloaded based on whether the local file path exists, not just what the DB says

Browse Source
This commit is contained in:
Alessio 2022-05-07 18:46:07 -07:00
parent 8775f5337d
commit 3d289ed5f0
3 changed files with 19 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
persistence/tweet_trove_queries.go
View File

@ -12,19 +12,19 @@ import (
*/ */
func (p Profile) SaveTweetTrove(trove TweetTrove) { func (p Profile) SaveTweetTrove(trove TweetTrove) {
for i, u := range trove.Users { for i, u := range trove.Users {
// Download download their tiny profile image err := p.SaveUser(&u)
err := p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
}
err = p.SaveUser(&u)
if err != nil { if err != nil {
panic(fmt.Errorf("Error saving user with ID %d and handle %s:\n %w", u.ID, u.Handle, err)) panic(fmt.Errorf("Error saving user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
} }
fmt.Println(u.Handle, u.ID) fmt.Println(u.Handle, u.ID)
// If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too // If the User's ID was updated in saving (i.e., Unknown User), update it in the Trove too
trove.Users[i] = u trove.Users[i] = u
// Download their tiny profile image
err = p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
}
} }
// TODO: this is called earlier in the process as well, before parsing. Is that call redundant? Too tired to figure out right now // TODO: this is called earlier in the process as well, before parsing. Is that call redundant? Too tired to figure out right now

7
persistence/user_queries.go
View File

@ -181,10 +181,13 @@ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
if !is_content_downloaded { if !is_content_downloaded {
return true return true
} }
if banner_image_url != user.BannerImageUrl {
banner_path := p.get_banner_image_output_path(user)
if banner_path != "" && !file_exists(banner_path) {
return true return true
} }
if profile_image_url != user.ProfileImageUrl { profile_path := p.get_profile_image_output_path(user)
if !file_exists(profile_path) {
return true return true
} }
return false return false

11
persistence/user_queries_test.go
View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"os"
"github.com/go-test/deep" "github.com/go-test/deep"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -164,13 +165,15 @@ func TestCheckUserContentDownloadNeeded(t *testing.T) {
err = profile.SaveUser(&user) err = profile.SaveUser(&user)
require.NoError(t, err) require.NoError(t, err)
// If everything is up to date, no download should be required // If the file exists, then it should not require any download
_, err = os.Create("test_profiles/TestUserQueries/profile_images/" + user.BannerImageLocalPath)
require.NoError(t, err)
_, err = os.Create("test_profiles/TestUserQueries/profile_images/" + user.ProfileImageLocalPath)
require.NoError(t, err)
assert.False(profile.CheckUserContentDownloadNeeded(user)) assert.False(profile.CheckUserContentDownloadNeeded(user))
// Change an URL, but don't save it-- needs to be different from what's in the DB
user.BannerImageUrl = "banner url2"
// Download needed for new banner image // Download needed for new banner image
user.BannerImageLocalPath = "banner url2"
assert.True(profile.CheckUserContentDownloadNeeded(user)) assert.True(profile.CheckUserContentDownloadNeeded(user))
} }