me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add to database: LocalPath version of BannerImage and ProfileImage on Users

Browse Source
This commit is contained in:
Alessio 2021-08-10 20:51:42 -07:00
parent 655c968614
commit 0a1fa92cdd
4 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/twitter/main.go
View File

@ -14,6 +14,8 @@ import (
var profile persistence.Profile var profile persistence.Profile
// TODO: use the current directory by default, add flag to set data-dir
/** /**
* Main method * Main method
*/ */

2
persistence/schema.sql
View File

@ -13,7 +13,9 @@ create table users (rowid integer primary key,
is_private boolean default 0, is_private boolean default 0,
is_verified boolean default 0, is_verified boolean default 0,
profile_image_url text, profile_image_url text,
profile_image_local_path text,
banner_image_url text, banner_image_url text,
banner_image_local_path text,
pinned_tweet_id integer check(typeof(pinned_tweet_id) = 'integer' or pinned_tweet_id = ''), pinned_tweet_id integer check(typeof(pinned_tweet_id) = 'integer' or pinned_tweet_id = ''),
is_content_downloaded boolean default 0 is_content_downloaded boolean default 0

15
persistence/user_queries.go
View File

@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error {
return err return err
} }
_, err = db.Exec(` _, err = db.Exec(`
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id) insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set bio=?, set bio=?,
following_count=?, following_count=?,
@ -32,10 +32,13 @@ func (p Profile) SaveUser(u scraper.User) error {
is_private=?, is_private=?,
is_verified=?, is_verified=?,
profile_image_url=?, profile_image_url=?,
profile_image_local_path=?,
banner_image_url=?, banner_image_url=?,
banner_image_local_path=?,
pinned_tweet_id=? pinned_tweet_id=?
`, `,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID, u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
) )
if err != nil { if err != nil {
return err return err
@ -81,7 +84,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) {
var u scraper.User var u scraper.User
var joinDate int64 var joinDate int64
err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.BannerImageUrl, &u.PinnedTweetID) err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID)
if err != nil { if err != nil {
return u, err return u, err
} }
@ -104,7 +107,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
from users from users
where handle = ? where handle = ?
`) `)
@ -135,7 +138,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
from users from users
where id = ? where id = ?
`) `)

4
persistence/utils_test.go
View File

@ -53,7 +53,9 @@ func create_stable_user() scraper.User {
IsVerified: true, IsVerified: true,
IsPrivate: false, IsPrivate: false,
ProfileImageUrl: "stable profile image url", ProfileImageUrl: "stable profile image url",
ProfileImageLocalPath: "stable profile image local path",
BannerImageUrl: "stable banner image url", BannerImageUrl: "stable banner image url",
BannerImageLocalPath: "stable image local path",
PinnedTweetID: scraper.TweetID(345), PinnedTweetID: scraper.TweetID(345),
} }
} }
@ -133,7 +135,9 @@ func create_dummy_user() scraper.User {
IsVerified: false, IsVerified: false,
IsPrivate: true, IsPrivate: true,
ProfileImageUrl: "profile image url", ProfileImageUrl: "profile image url",
ProfileImageLocalPath: "profile image local path",
BannerImageUrl: "banner image url", BannerImageUrl: "banner image url",
BannerImageLocalPath: "banner image local path",
PinnedTweetID: scraper.TweetID(234), PinnedTweetID: scraper.TweetID(234),
} }
} }