me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add to database: LocalPath version of BannerImage and ProfileImage on Users

Browse Source
This commit is contained in:
Alessio 2021-08-10 20:51:42 -07:00
parent 655c968614
commit 0a1fa92cdd
4 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/twitter/main.go
View File

@ -14,6 +14,8 @@ import (
var profile persistence.Profile
// TODO: use the current directory by default, add flag to set data-dir
/**
* Main method
*/

2
persistence/schema.sql
View File

@ -13,7 +13,9 @@ create table users (rowid integer primary key,
is_private boolean default 0,
is_verified boolean default 0,
profile_image_url text,
profile_image_local_path text,
banner_image_url text,
banner_image_local_path text,
pinned_tweet_id integer check(typeof(pinned_tweet_id) = 'integer' or pinned_tweet_id = ''),
is_content_downloaded boolean default 0

15
persistence/user_queries.go
View File

@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error {
return err
}
_, err = db.Exec(`
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update
set bio=?,
following_count=?,
@ -32,10 +32,13 @@ func (p Profile) SaveUser(u scraper.User) error {
is_private=?,
is_verified=?,
profile_image_url=?,
profile_image_local_path=?,
banner_image_url=?,
banner_image_local_path=?,
pinned_tweet_id=?
`,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
)
if err != nil {
return err
@ -81,7 +84,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) {
var u scraper.User
var joinDate int64
err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.BannerImageUrl, &u.PinnedTweetID)
err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID)
if err != nil {
return u, err
}
@ -104,7 +107,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error
db := p.DB
stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
from users
where handle = ?
`)
@ -135,7 +138,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
db := p.DB
stmt, err := db.Prepare(`
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id
select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id
from users
where id = ?
`)

4
persistence/utils_test.go
View File

@ -53,7 +53,9 @@ func create_stable_user() scraper.User {
IsVerified: true,
IsPrivate: false,
ProfileImageUrl: "stable profile image url",
ProfileImageLocalPath: "stable profile image local path",
BannerImageUrl: "stable banner image url",
BannerImageLocalPath: "stable image local path",
PinnedTweetID: scraper.TweetID(345),
}
}
@ -133,7 +135,9 @@ func create_dummy_user() scraper.User {
IsVerified: false,
IsPrivate: true,
ProfileImageUrl: "profile image url",
ProfileImageLocalPath: "profile image local path",
BannerImageUrl: "banner image url",
BannerImageLocalPath: "banner image local path",
PinnedTweetID: scraper.TweetID(234),
}
}