diff --git a/cmd/tests.sh b/cmd/tests.sh index 1b54ab9..9e49bde 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -108,6 +108,16 @@ test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 14295854 test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice" +# Test that profile images (tiny vs regular) are chosen properly +test $(sqlite3 twitter.db "select is_content_downloaded from users where handle = 'Cernovich'") = "1" +test $(find profile_images/Cernovich* | grep normal | wc -l) = "0" # Since "Cernovich" was fetched directly, should have full-sized profile image and banner +test $(find profile_images/Cernovich* | grep banner | wc -l) = "1" + +test $(sqlite3 twitter.db "select is_content_downloaded from users where handle = 'RememberAfghan1'") = "0" +test $(find profile_images/RememberAfghan1* | grep normal | wc -l) = "1" # "RememberAfghan1" was fetched via a tweet thread and isn't followed, so should have tiny profile image and no banner +test $(find profile_images/RememberAfghan1* | grep banner | wc -l) = "0" + + # Test that the `--profile` flag works cd .. tw --profile data fetch_user elonmusk diff --git a/persistence/media_download.go b/persistence/media_download.go index 79bf0ae..649f49a 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -170,6 +170,21 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med return p.SaveUser(*u) } +/** + * Download a User's tiny profile image, if it hasn't been downloaded yet. + * If it has been downloaded, do nothing. + */ +func (p Profile) DownloadUserProfileImageTiny(u scraper.User) error { + d := DefaultDownloader{} + + outfile := path.Join(p.ProfileDir, "profile_images", u.GetTinyProfileImageLocalPath()) + if file_exists(outfile) { + return nil + } + err := d.Curl(u.GetTinyProfileImageUrl(), outfile) + return err +} + /** * Download a User's content, if needed. @@ -180,7 +195,12 @@ func (p Profile) DownloadUserContentIfNeeded(u *scraper.User) (bool, error) { if !p.CheckUserContentDownloadNeeded(*u) { return false, nil } - return true, p.DownloadUserContentFor(u) + if p.IsFollowing(u.Handle) { + // TODO: this might not be necessary? When would someone be followed but content not downloaded? + return true, p.DownloadUserContentFor(u) + } else { + return true, p.DownloadUserProfileImageTiny(*u) + } } diff --git a/scraper/user.go b/scraper/user.go index 565c104..ad11e03 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -160,3 +160,7 @@ func (u User) GetTinyProfileImageUrl() string { } return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1") } + +func (u User) GetTinyProfileImageLocalPath() string { + return string(u.Handle) + "_profile_" + path.Base(u.GetTinyProfileImageUrl()) +}