Allow fetching of banned users

cmd/tests.sh

@@ -207,6 +207,11 @@ test "$(sqlite3 twitter.db "select choice1, choice2, choice3, choice4 from polls
 test "$(sqlite3 twitter.db "select choice1_votes, choice2_votes, choice3_votes, choice4_votes from polls where tweet_id = 1465534109573390348")" = "1593|624|778|1138"
 
 
+# Test fetching a banned user
+test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='kanesays23'") = "0"
+tw fetch_user kanesays23
+test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='kanesays23'")" = "1|1"
+
 # TODO: Maybe this file should be broken up into multiple test scripts
 
 echo -e "\033[32mAll tests passed.  Finished successfully.\033[0m"

persistence/media_download.go

@@ -147,8 +147,18 @@ func (p Profile) DownloadUserContentFor(u *scraper.User) error {
  * Enable injecting a custom MediaDownloader (i.e., for testing)
  */
 func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error {
-    outfile := path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath)
+    var outfile string
-    err := downloader.Curl(u.ProfileImageUrl, outfile)
+    var target_url string
+
+    if u.ProfileImageUrl == "" {
+        outfile = path.Join(p.ProfileDir, "profile_images", path.Base(scraper.DEFAULT_PROFILE_IMAGE_URL))
+        target_url = scraper.DEFAULT_PROFILE_IMAGE_URL
+    } else {
+        outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath)
+        target_url = u.ProfileImageUrl
+    }
+
+    err := downloader.Curl(target_url, outfile)
     if err != nil {
         return err
     }

persistence/user_queries.go

@@ -152,6 +152,8 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
 /**
  * Returns `true` if content download is needed, `false` otherwise
  *
+ * If the user is banned, returns false because downloading will be impossible.
+ *
  * If:
  * - the user isn't in the DB at all (first time scraping), OR
  * - `is_content_downloaded` is false in the DB, OR
@@ -162,6 +164,10 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
  * why the No Worsening Principle is needed.
  */
 func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
+    if user.IsBanned {
+        // Check `is_banned` on the live user, since he may have been un-banned since last scraped
+        return false
+    }
     row := p.DB.QueryRow(`select is_content_downloaded, profile_image_url, banner_image_url from users where id = ?`, user.ID)
 
     var is_content_downloaded bool

scraper/api_request_utils.go

@@ -208,12 +208,15 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) {
 			return APIUser{}, err
 		}
 
-	    if len(response.Errors) == 0 {
+		// Retry ONLY if the error is code 50 (random authentication failure), NOT on real errors
-	        break
+		if len(response.Errors) == 1 && response.Errors[0].Code == 50 {
-	    }
-
 			// Reset the response (remove the Errors)
 			response = UserResponse{}
+			continue
+		} else {
+			// Do not retry on real errors
+			break
+		}
 	}
 	return response.ConvertToAPIUser(), err
 }