me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix some problems associated with banned users

Browse Source
This commit is contained in:
Alessio 2022-01-07 16:23:02 -05:00
parent c253486580
commit 12f1cc6728
3 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/tests.sh
View File

@ -97,12 +97,14 @@ test $(find videos | wc -l) = "$((initial_videos_count + 1))"
# Download a full thread
test ! -e profile_images/default_profile_normal.png
tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1"
test $(sqlite3 twitter.db "select is_conversation_scraped, abs(last_scraped_at - strftime('%s','now')) < 30 from tweets where id = 1429585423702052867") = "1|1"
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice"
test $(sqlite3 twitter.db "select is_conversation_scraped from tweets where id = 1429584239570391042") = "0"
test "$(sqlite3 twitter.db "select handle, is_banned from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730")" = "kanesays23|1" # This guy got banned
test -e profile_images/default_profile_normal.png
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice"
test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice"
@ -208,9 +210,10 @@ test "$(sqlite3 twitter.db "select choice1_votes, choice2_votes, choice3_votes,
# Test fetching a banned user
test $(sqlite3 twitter.db "select is_content_downloaded from users where handle='kanesays23'") = "0"
tw fetch_user kanesays23
test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='kanesays23'")" = "1|1"
test ! -e profile_images/default_profile.png
tw fetch_user nancytracker
test "$(sqlite3 twitter.db "select is_content_downloaded, is_banned from users where handle='nancytracker'")" = "1|1"
test -e profile_images/default_profile.png
# TODO: Maybe this file should be broken up into multiple test scripts

4
persistence/user_queries.go
View File

@ -164,10 +164,6 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
* why the No Worsening Principle is needed.
*/
func (p Profile) CheckUserContentDownloadNeeded(user scraper.User) bool {
if user.IsBanned {
// Check `is_banned` on the live user, since he may have been un-banned since last scraped
return false
}
row := p.DB.QueryRow(`select is_content_downloaded, profile_image_url, banner_image_url from users where id = ?`, user.ID)
var is_content_downloaded bool

5
scraper/user.go
View File

@ -98,13 +98,13 @@ func ParseHandleFromTweetUrl(tweet_url string) (UserHandle, error) {
// Turn an APIUser, as returned from the scraper, into a properly structured User object
func ParseSingleUser(apiUser APIUser) (ret User, err error) {
ret.ID = UserID(apiUser.ID)
ret.Handle = UserHandle(apiUser.ScreenName)
if apiUser.IsBanned {
// Banned users won't have any further info, so just return here
ret.IsBanned = true
return
}
ret.DisplayName = apiUser.Name
ret.Handle = UserHandle(apiUser.ScreenName)
ret.Bio = apiUser.Description
ret.FollowingCount = apiUser.FriendsCount
ret.FollowersCount = apiUser.FollowersCount
@ -139,6 +139,9 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) {
func GetUser(handle UserHandle) (User, error) {
api := API{}
apiUser, err := api.GetUser(handle)
if apiUser.ScreenName == "" {
apiUser.ScreenName = string(handle)
}
if err != nil {
return User{}, err
}