diff --git a/cmd/tests.sh b/cmd/tests.sh index da41599..522648a 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -97,12 +97,14 @@ test $(find videos | wc -l) = "$((initial_videos_count + 1))" # Download a full thread +test ! -e profile_images/default_profile_normal.png tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867 test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1" test $(sqlite3 twitter.db "select is_conversation_scraped, abs(last_scraped_at - strftime('%s','now')) < 30 from tweets where id = 1429585423702052867") = "1|1" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice" test $(sqlite3 twitter.db "select is_conversation_scraped from tweets where id = 1429584239570391042") = "0" test "$(sqlite3 twitter.db "select handle, is_banned from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730")" = "kanesays23|1" # This guy got banned +test -e profile_images/default_profile_normal.png test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis" test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice" test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice" @@ -208,9 +210,10 @@ test "$(sqlite3 twitter.db "select choice1_votes, choice2_votes, choice3_votes, # Test fetching a banned user -test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='kanesays23'") = "0" -tw fetch_user kanesays23 -test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='kanesays23'")" = "1|1" +test ! -e profile_images/default_profile.png +tw fetch_user nancytracker +test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='nancytracker'")" = "1|1" +test -e profile_images/default_profile.png # TODO: Maybe this file should be broken up into multiple test scripts diff --git a/persistence/user_queries.go b/persistence/user_queries.go index 75c3a05..e5208b1 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -164,10 +164,6 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) { * why the No Worsening Principle is needed. */ func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool { - if user.IsBanned { - // Check `is_banned` on the live user, since he may have been un-banned since last scraped - return false - } row := p.DB.QueryRow(`select is_content_downloaded, profile_image_url, banner_image_url from users where id = ?`, user.ID) var is_content_downloaded bool diff --git a/scraper/user.go b/scraper/user.go index f6fe5f8..0eae868 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -98,13 +98,13 @@ func ParseHandleFromTweetUrl(tweet_url string) (UserHandle, error) { // Turn an APIUser, as returned from the scraper, into a properly structured User object func ParseSingleUser(apiUser APIUser) (ret User, err error) { ret.ID = UserID(apiUser.ID) + ret.Handle = UserHandle(apiUser.ScreenName) if apiUser.IsBanned { // Banned users won't have any further info, so just return here ret.IsBanned = true return } ret.DisplayName = apiUser.Name - ret.Handle = UserHandle(apiUser.ScreenName) ret.Bio = apiUser.Description ret.FollowingCount = apiUser.FriendsCount ret.FollowersCount = apiUser.FollowersCount @@ -139,6 +139,9 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) { func GetUser(handle UserHandle) (User, error) { api := API{} apiUser, err := api.GetUser(handle) + if apiUser.ScreenName == "" { + apiUser.ScreenName = string(handle) + } if err != nil { return User{}, err }