Fix some problems associated with banned users

cmd/tests.sh

@@ -97,12 +97,14 @@ test $(find videos | wc -l) = "$((initial_videos_count + 1))"
 
 
 # Download a full thread
+test ! -e profile_images/default_profile_normal.png
 tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867
 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1"
 test $(sqlite3 twitter.db "select is_conversation_scraped, abs(last_scraped_at - strftime('%s','now')) < 30 from tweets where id = 1429585423702052867") = "1|1"
 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice"
 test $(sqlite3 twitter.db "select is_conversation_scraped from tweets where id = 1429584239570391042") = "0"
 test "$(sqlite3 twitter.db "select handle, is_banned from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730")" = "kanesays23|1"  # This guy got banned
+test -e profile_images/default_profile_normal.png
 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis"
 test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice"
 test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice"
@@ -208,9 +210,10 @@ test "$(sqlite3 twitter.db "select choice1_votes, choice2_votes, choice3_votes,
 
 
 # Test fetching a banned user
-test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='kanesays23'") = "0"
+test ! -e profile_images/default_profile.png
-tw fetch_user kanesays23
+tw fetch_user nancytracker
-test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='kanesays23'")" = "1|1"
+test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='nancytracker'")" = "1|1"
+test -e profile_images/default_profile.png
 
 # TODO: Maybe this file should be broken up into multiple test scripts
 

persistence/user_queries.go

@@ -164,10 +164,6 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
  * why the No Worsening Principle is needed.
  */
 func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
-    if user.IsBanned {
-        // Check `is_banned` on the live user, since he may have been un-banned since last scraped
-        return false
-    }
     row := p.DB.QueryRow(`select is_content_downloaded, profile_image_url, banner_image_url from users where id = ?`, user.ID)
 
     var is_content_downloaded bool

scraper/user.go

@@ -98,13 +98,13 @@ func ParseHandleFromTweetUrl(tweet_url string) (UserHandle, error) {
 // Turn an APIUser, as returned from the scraper, into a properly structured User object
 func ParseSingleUser(apiUser APIUser) (ret User, err error) {
     ret.ID = UserID(apiUser.ID)
+    ret.Handle = UserHandle(apiUser.ScreenName)
     if apiUser.IsBanned {
         // Banned users won't have any further info, so just return here
         ret.IsBanned = true
         return
     }
     ret.DisplayName = apiUser.Name
-    ret.Handle = UserHandle(apiUser.ScreenName)
     ret.Bio = apiUser.Description
     ret.FollowingCount = apiUser.FriendsCount
     ret.FollowersCount = apiUser.FollowersCount
@@ -139,6 +139,9 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) {
 func GetUser(handle UserHandle) (User, error) {
     api := API{}
     apiUser, err := api.GetUser(handle)
+    if apiUser.ScreenName == "" {
+        apiUser.ScreenName = string(handle)
+    }
     if err != nil {
         return User{}, err
     }