From 3d489a8e95677b82969bdbadd2e6337f50b97bf2 Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 14 Feb 2022 13:10:12 -0800 Subject: [PATCH] REFACTOR: eliminate 'download-if-needed' functions, add the 'if-needed' check to the regular downloading functions --- cmd/twitter/main.go | 12 ++++----- persistence/media_download.go | 47 +++++++++++------------------------ 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index f38cce3..62d59b4 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -184,7 +184,7 @@ func fetch_tweet_conversation(tweet_identifier string) { for _, u := range users { fmt.Println(u.Handle) - _, err = profile.DownloadUserContentIfNeeded(&u) + err = profile.DownloadUserProfileImageTiny(&u) if err != nil { die("Error getting user content: " + err.Error(), false, 10) } @@ -200,7 +200,7 @@ func fetch_tweet_conversation(tweet_identifier string) { if err != nil { die(fmt.Sprintf("Error saving tweet (id %d): %s", t.ID, err.Error()), false, 4) } - _, err = profile.DownloadTweetContentIfNeeded(&t) + err = profile.DownloadTweetContentFor(&t) if err != nil { die("Error getting tweet content: " + err.Error(), false, 11) } @@ -229,7 +229,7 @@ func fetch_user_feed(handle string, how_many int) { for _, u := range users { fmt.Println(u.Handle) - _, err = profile.DownloadUserContentIfNeeded(&u) + err = profile.DownloadUserProfileImageTiny(&u) if err != nil { die("Error getting user content: " + err.Error(), false, 10) } @@ -244,7 +244,7 @@ func fetch_user_feed(handle string, how_many int) { if err != nil { die("Error saving tweet: " + err.Error(), false, 4) } - _, err = profile.DownloadTweetContentIfNeeded(&t) + err = profile.DownloadTweetContentFor(&t) if err != nil { die("Error getting tweet content: " + err.Error(), false, 11) } @@ -298,7 +298,7 @@ func search(query string) { for _, u := range users { fmt.Println(u.Handle) - _, err = profile.DownloadUserContentIfNeeded(&u) + err = profile.DownloadUserProfileImageTiny(&u) if err != nil { die("Error getting user content: " + err.Error(), false, 10) } @@ -314,7 +314,7 @@ func search(query string) { if err != nil { die("Error saving tweet: " + err.Error(), false, 4) } - _, err = profile.DownloadTweetContentIfNeeded(&t) + err = profile.DownloadTweetContentFor(&t) if err != nil { die("Error getting tweet content: " + err.Error(), false, 11) } diff --git a/persistence/media_download.go b/persistence/media_download.go index 38881cb..e0395bf 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -112,6 +112,11 @@ func (p Profile) DownloadTweetContentFor(t *scraper.Tweet) error { * Enable injecting a custom MediaDownloader (i.e., for testing) */ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader MediaDownloader) error { + // Check if content needs to be downloaded; if not, just return + if !p.CheckTweetContentDownloadNeeded(*t) { + return nil + } + for i := range t.Images { err := p.download_tweet_image(&t.Images[i], downloader) if err != nil { @@ -147,6 +152,10 @@ func (p Profile) DownloadUserContentFor(u *scraper.User) error { * Enable injecting a custom MediaDownloader (i.e., for testing) */ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error { + if !p.CheckUserContentDownloadNeeded(*u) { + return nil + } + var outfile string var target_url string @@ -167,6 +176,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med if u.BannerImageLocalPath != "" { outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath) err = downloader.Curl(u.BannerImageUrl, outfile) + if err != nil && strings.Contains(err.Error(), "404 Not Found") { // Try adding "600x200". Not sure why this does this but sometimes it does. err = downloader.Curl(u.BannerImageUrl + "/600x200", outfile) @@ -184,7 +194,11 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med * Download a User's tiny profile image, if it hasn't been downloaded yet. * If it has been downloaded, do nothing. */ -func (p Profile) DownloadUserProfileImageTiny(u scraper.User) error { +func (p Profile) DownloadUserProfileImageTiny(u *scraper.User) error { + if p.IsFollowing(u.Handle) { + return p.DownloadUserContentFor(u) + } + d := DefaultDownloader{} outfile := path.Join(p.ProfileDir, "profile_images", u.GetTinyProfileImageLocalPath()) @@ -194,34 +208,3 @@ func (p Profile) DownloadUserProfileImageTiny(u scraper.User) error { err := d.Curl(u.GetTinyProfileImageUrl(), outfile) return err } - - -/** - * Download a User's content, if needed. - * - * Returns whether anything was downloaded or not. - */ -func (p Profile) DownloadUserContentIfNeeded(u *scraper.User) (bool, error) { - if !p.CheckUserContentDownloadNeeded(*u) { - return false, nil - } - if p.IsFollowing(u.Handle) { - // TODO: this might not be necessary? When would someone be followed but content not downloaded? - return true, p.DownloadUserContentFor(u) - } else { - return true, p.DownloadUserProfileImageTiny(*u) - } -} - - -/** - * Download a Tweet's content, if needed. - * - * Returns whether anything was downloaded or not. - */ -func (p Profile) DownloadTweetContentIfNeeded(t *scraper.Tweet) (bool, error) { - if !p.CheckTweetContentDownloadNeeded(*t) { - return false, nil - } - return true, p.DownloadTweetContentFor(t) -}