me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add downloading of User banner image and profile image

Browse Source
This commit is contained in:
Alessio 2021-08-10 21:35:27 -07:00
parent 0a1fa92cdd
commit 294917124a
3 changed files with 85 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
persistence/media_download.go
View File

@ -102,3 +102,36 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M
t.IsContentDownloaded = true
return p.SaveTweet(*t)
}
/**
* Download a user's banner and profile images
*/
func (p Profile) DownloadUserContentFor(u *scraper.User) error {
return p.DownloadUserContentWithInjector(u, DefaultDownloader{})
}
/**
* Enable injecting a custom MediaDownloader (i.e., for testing)
*/
func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error {
var err error
var outfile string
outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath)
err = downloader.Curl(u.ProfileImageUrl, outfile)
if err != nil {
return err
}
// Skip it if there's no banner image
if u.BannerImageLocalPath != "" {
outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath)
err = downloader.Curl(u.BannerImageUrl, outfile)
if err != nil {
return err
}
}
u.IsContentDownloaded = true
return p.SaveUser(*u)
}

46
persistence/media_download_test.go
View File

@ -37,7 +37,7 @@ func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) {
}
/**
* Create an Image, save it, reload it, and make sure it comes back the same
* Downloading a Tweet's contents should mark the Tweet as downloaded
*/
func TestDownloadTweetContent(t *testing.T) {
profile_path := "test_profiles/TestMediaQueries"
@ -63,10 +63,50 @@ func TestDownloadTweetContent(t *testing.T) {
// It should all be marked "yes downloaded" now
test_all_downloaded(tweet, true, t)
// Reload the tweet (check db); should also be "yes downloaded"
// Reload the Tweet (check db); should also be "yes downloaded"
new_tweet, err := profile.GetTweetById(tweet.ID)
if err != nil {
t.Fatalf("Couldn't reload the tweet: %s", err.Error())
t.Fatalf("Couldn't reload the Tweet: %s", err.Error())
}
test_all_downloaded(new_tweet, true, t)
}
/**
* Downloading a User's contents should mark the User as downloaded
*/
func TestDownloadUserContent(t *testing.T) {
profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path)
user := create_dummy_user()
// Persist the User
err := profile.SaveUser(user)
if err != nil {
t.Fatalf("Failed to save the user: %s", err.Error())
}
// Make sure the User is marked "not downloaded"
if user.IsContentDownloaded {
t.Errorf("User shouldn't be marked downloaded, but it was")
}
// Do the (fake) downloading
err = profile.DownloadUserContentWithInjector(&user, FakeDownloader{})
if err != nil {
t.Fatalf("Error running fake download: %s", err.Error())
}
// The User should now be marked "yes downloaded"
if !user.IsContentDownloaded {
t.Errorf("User should be marked downloaded, but it wasn't")
}
// Reload the User (check db); should also be "yes downloaded"
new_user, err := profile.GetUserByID(user.ID)
if err != nil {
t.Fatalf("Couldn't reload the User: %s", err.Error())
}
if !new_user.IsContentDownloaded {
t.Errorf("User should be marked downloaded, but it wasn't")
}}

17
persistence/user_queries.go
View File

@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error {
return err
}
_, err = db.Exec(`
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update
set bio=?,
following_count=?,
@ -35,10 +35,11 @@ func (p Profile) SaveUser(u scraper.User) error {
profile_image_local_path=?,
banner_image_url=?,
banner_image_local_path=?,
pinned_tweet_id=?
pinned_tweet_id=?,
is_content_downloaded=?
`,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
)
if err != nil {
return err
@ -84,7 +85,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) {
var u scraper.User
var joinDate int64
err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID)
err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID, &u.IsContentDownloaded)
if err != nil {
return u, err
}
@ -107,7 +108,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
db := p.DB
stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded
from users
where handle = ?
`)
@ -138,7 +139,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
db := p.DB
stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded
from users
where id = ?
`)