diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 0fc64f4..f455bb1 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -14,6 +14,8 @@ import ( var profile persistence.Profile +// TODO: use the current directory by default, add flag to set data-dir + /** * Main method */ diff --git a/persistence/schema.sql b/persistence/schema.sql index ed1cece..5d32825 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -13,7 +13,9 @@ create table users (rowid integer primary key, is_private boolean default 0, is_verified boolean default 0, profile_image_url text, + profile_image_local_path text, banner_image_url text, + banner_image_local_path text, pinned_tweet_id integer check(typeof(pinned_tweet_id) = 'integer' or pinned_tweet_id = ''), is_content_downloaded boolean default 0 diff --git a/persistence/user_queries.go b/persistence/user_queries.go index ac3e7b7..26439d8 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -21,8 +21,8 @@ func (p Profile) SaveUser(u scraper.User) error { return err } _, err = db.Exec(` - insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set bio=?, following_count=?, @@ -32,10 +32,13 @@ func (p Profile) SaveUser(u scraper.User) error { is_private=?, is_verified=?, profile_image_url=?, + profile_image_local_path=?, banner_image_url=?, + banner_image_local_path=?, pinned_tweet_id=? `, - u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.BannerImageUrl, u.PinnedTweetID, + u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, + u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, ) if err != nil { return err @@ -81,7 +84,7 @@ func parse_user_from_row(row *sql.Row) (scraper.User, error) { var u scraper.User var joinDate int64 - err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.BannerImageUrl, &u.PinnedTweetID) + err := row.Scan(&u.ID, &u.DisplayName, &u.Handle, &u.Bio, &u.FollowingCount, &u.FollowersCount, &u.Location, &u.Website, &joinDate, &u.IsPrivate, &u.IsVerified, &u.ProfileImageUrl, &u.ProfileImageLocalPath, &u.BannerImageUrl, &u.BannerImageLocalPath, &u.PinnedTweetID) if err != nil { return u, err } @@ -104,7 +107,7 @@ func (p Profile) GetUserByHandle(handle scraper.UserHandle) (scraper.User, error db := p.DB stmt, err := db.Prepare(` - select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id + select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id from users where handle = ? `) @@ -135,7 +138,7 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) { db := p.DB stmt, err := db.Prepare(` - select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, banner_image_url, pinned_tweet_id + select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id from users where id = ? `) diff --git a/persistence/utils_test.go b/persistence/utils_test.go index 427ec81..51a6f46 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -53,7 +53,9 @@ func create_stable_user() scraper.User { IsVerified: true, IsPrivate: false, ProfileImageUrl: "stable profile image url", + ProfileImageLocalPath: "stable profile image local path", BannerImageUrl: "stable banner image url", + BannerImageLocalPath: "stable image local path", PinnedTweetID: scraper.TweetID(345), } } @@ -133,7 +135,9 @@ func create_dummy_user() scraper.User { IsVerified: false, IsPrivate: true, ProfileImageUrl: "profile image url", + ProfileImageLocalPath: "profile image local path", BannerImageUrl: "banner image url", + BannerImageLocalPath: "banner image local path", PinnedTweetID: scraper.TweetID(234), } }