From cae95ed99d557b4cf0094bcfe2c041bee651a1d0 Mon Sep 17 00:00:00 2001 From: Alessio Date: Tue, 27 Jun 2023 21:56:29 -0300 Subject: [PATCH] REFACTOR: change all appropriate queries to use sqlx --- persistence/media_queries.go | 58 +++++++++++++++------------------- persistence/retweet_queries.go | 6 ++-- persistence/user_queries.go | 45 ++++++++++++-------------- scraper/image.go | 14 ++++---- scraper/poll.go | 26 +++++++-------- scraper/retweet.go | 6 ++-- scraper/url.go | 31 +++++++++--------- scraper/user.go | 42 ++++++++++++------------ scraper/video.go | 24 +++++++------- 9 files changed, 121 insertions(+), 131 deletions(-) diff --git a/persistence/media_queries.go b/persistence/media_queries.go index e85c396..98e6431 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -13,14 +13,13 @@ import ( * - img: the Image to save */ func (p Profile) SaveImage(img scraper.Image) error { - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded) - values (?, ?, ?, ?, ?, ?, ?) + values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :is_downloaded) on conflict do update - set is_downloaded=(is_downloaded or ?) + set is_downloaded=(is_downloaded or :is_downloaded) `, - img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded, - img.IsDownloaded, + img, ) if err != nil { return fmt.Errorf("Error saving image (tweet ID %d):\n %w", img.TweetID, err) @@ -35,19 +34,17 @@ func (p Profile) SaveImage(img scraper.Image) error { * - img: the Video to save */ func (p Profile) SaveVideo(vid scraper.Video) error { - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url, :thumbnail_local_filename, + :duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif) on conflict do update - set is_downloaded=(is_downloaded or ?), - view_count=max(view_count, ?), - is_blocked_by_dmca = ? + set is_downloaded=(is_downloaded or :is_downloaded), + view_count=max(view_count, :view_count), + is_blocked_by_dmca = :is_blocked_by_dmca `, - vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath, - vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsBlockedByDMCA, vid.IsGif, - - vid.IsDownloaded, vid.ViewCount, vid.IsBlockedByDMCA, + vid, ) if err != nil { return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err) @@ -59,17 +56,16 @@ func (p Profile) SaveVideo(vid scraper.Video) error { * Save an Url */ func (p Profile) SaveUrl(url scraper.Url) error { - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + values (:tweet_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width, + :thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded + ) on conflict do update - set is_content_downloaded=(is_content_downloaded or ?) + set is_content_downloaded=(is_content_downloaded or :is_content_downloaded) `, - url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth, - url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded, - - url.IsContentDownloaded, + url, ) if err != nil { return fmt.Errorf("Error saving Url (tweet ID %d):\n %w", url.TweetID, err) @@ -81,21 +77,19 @@ func (p Profile) SaveUrl(url scraper.Url) error { * Save a Poll */ func (p Profile) SavePoll(poll scraper.Poll) error { - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, voting_duration, voting_ends_at, last_scraped_at) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + values (:id, :tweet_id, :num_choices, :choice1, :choice1_votes, :choice2, :choice2_votes, :choice3, :choice3_votes, + :choice4, :choice4_votes, :voting_duration, :voting_ends_at, :last_scraped_at) on conflict do update - set choice1_votes=?, - choice2_votes=?, - choice3_votes=?, - choice4_votes=?, - last_scraped_at=? + set choice1_votes=:choice1_votes, + choice2_votes=:choice2_votes, + choice3_votes=:choice3_votes, + choice4_votes=:choice4_votes, + last_scraped_at=:last_scraped_at `, - poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3, - poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt, poll.LastUpdatedAt, - - poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt, + poll, ) if err != nil { return fmt.Errorf("Error saving Poll (tweet ID %d):\n %w", poll.TweetID, err) diff --git a/persistence/retweet_queries.go b/persistence/retweet_queries.go index 78d3d37..c236ace 100644 --- a/persistence/retweet_queries.go +++ b/persistence/retweet_queries.go @@ -10,12 +10,12 @@ import ( * Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable. */ func (p Profile) SaveRetweet(r scraper.Retweet) error { - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at) - values (?, ?, ?, ?) + values (:retweet_id, :tweet_id, :retweeted_by, :retweeted_at) on conflict do nothing `, - r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(), + r, ) if err != nil { return fmt.Errorf("Error executing SaveRetweet(%#v):\n %w", r, err) diff --git a/persistence/user_queries.go b/persistence/user_queries.go index 4323102..c78b3ee 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -33,35 +33,32 @@ func (p Profile) SaveUser(u *scraper.User) error { } } - _, err := p.DB.Exec(` + _, err := p.DB.NamedExec(` insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded, is_id_fake) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + values (:id, :display_name, :handle, :bio, :following_count, :followers_count, :location, :website, :join_date, :is_private, + :is_verified, :is_banned, :profile_image_url, :profile_image_local_path, :banner_image_url, :banner_image_local_path, + :pinned_tweet_id, :is_content_downloaded, :is_id_fake) on conflict do update - set handle=?, - bio=?, - display_name=?, - following_count=?, - followers_count=?, - location=?, - website=?, - is_private=?, - is_verified=?, - is_banned=?, - profile_image_url=?, - profile_image_local_path=?, - banner_image_url=?, - banner_image_local_path=?, - pinned_tweet_id=?, - is_content_downloaded=(is_content_downloaded or ?) + set handle=:handle, + bio=:bio, + display_name=:display_name, + following_count=:following_count, + followers_count=:followers_count, + location=:location, + website=:website, + is_private=:is_private, + is_verified=:is_verified, + is_banned=:is_banned, + profile_image_url=:profile_image_url, + profile_image_local_path=:profile_image_local_path, + banner_image_url=:banner_image_url, + banner_image_local_path=:banner_image_local_path, + pinned_tweet_id=:pinned_tweet_id, + is_content_downloaded=(is_content_downloaded or :is_content_downloaded) `, - u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate, u.IsPrivate, - u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, - u.IsContentDownloaded, u.IsIdFake, - - u.Handle, u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned, - u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, + u, ) if err != nil { return fmt.Errorf("Error executing SaveUser(%s):\n %w", u.Handle, err) diff --git a/scraper/image.go b/scraper/image.go index f13031f..b17e192 100644 --- a/scraper/image.go +++ b/scraper/image.go @@ -7,13 +7,13 @@ import ( type ImageID int64 type Image struct { - ID ImageID - TweetID TweetID - Width int - Height int - RemoteURL string - LocalFilename string - IsDownloaded bool + ID ImageID `db:"id"` + TweetID TweetID `db:"tweet_id"` + Width int `db:"width"` + Height int `db:"height"` + RemoteURL string `db:"remote_url"` + LocalFilename string `db:"local_filename"` + IsDownloaded bool `db:"is_downloaded"` } func ParseAPIMedia(apiMedia APIMedia) Image { diff --git a/scraper/poll.go b/scraper/poll.go index eb6de0e..0599744 100644 --- a/scraper/poll.go +++ b/scraper/poll.go @@ -9,21 +9,21 @@ import ( type PollID int64 type Poll struct { - ID PollID - TweetID TweetID - NumChoices int + ID PollID `db:"id"` + TweetID TweetID `db:"tweet_id"` + NumChoices int `db:"num_choices"` - Choice1 string - Choice1_Votes int - Choice2 string - Choice2_Votes int - Choice3 string - Choice3_Votes int - Choice4 string - Choice4_Votes int + Choice1 string `db:"choice1"` + Choice1_Votes int `db:"choice1_votes"` + Choice2 string `db:"choice2"` + Choice2_Votes int `db:"choice2_votes"` + Choice3 string `db:"choice3"` + Choice3_Votes int `db:"choice3_votes"` + Choice4 string `db:"choice4"` + Choice4_Votes int `db:"choice4_votes"` - VotingDuration int // In seconds - VotingEndsAt Timestamp + VotingDuration int `db:"voting_duration"` // In seconds + VotingEndsAt Timestamp `db:"voting_ends_at"` LastUpdatedAt Timestamp `db:"last_scraped_at"` } diff --git a/scraper/retweet.go b/scraper/retweet.go index a4ca028..4e1e1da 100644 --- a/scraper/retweet.go +++ b/scraper/retweet.go @@ -1,12 +1,12 @@ package scraper type Retweet struct { - RetweetID TweetID - TweetID TweetID + RetweetID TweetID `db:"retweet_id"` + TweetID TweetID `db:"tweet_id"` Tweet *Tweet RetweetedByID UserID `db:"retweeted_by"` RetweetedBy *User - RetweetedAt Timestamp + RetweetedAt Timestamp `db:"retweeted_at"` } func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) { diff --git a/scraper/url.go b/scraper/url.go index ec77f49..80334a5 100644 --- a/scraper/url.go +++ b/scraper/url.go @@ -8,23 +8,22 @@ import ( ) type Url struct { - TweetID TweetID + TweetID TweetID `db:"tweet_id"` + Domain string `db:"domain"` + Text string `db:"text"` + ShortText string `db:"short_text"` + Title string `db:"title"` + Description string `db:"description"` + ThumbnailWidth int `db:"thumbnail_width"` + ThumbnailHeight int `db:"thumbnail_height"` + ThumbnailRemoteUrl string `db:"thumbnail_remote_url"` + ThumbnailLocalPath string `db:"thumbnail_local_path"` + CreatorID UserID `db:"creator_id"` + SiteID UserID `db:"site_id"` - Domain string - Text string - ShortText string - Title string - Description string - ThumbnailWidth int - ThumbnailHeight int - ThumbnailRemoteUrl string - ThumbnailLocalPath string - CreatorID UserID - SiteID UserID - - HasCard bool - HasThumbnail bool - IsContentDownloaded bool + HasCard bool `db:"has_card"` + HasThumbnail bool `db:"has_thumbnail"` + IsContentDownloaded bool `db:"is_content_downloaded"` } func ParseAPIUrlCard(apiCard APICard) Url { diff --git a/scraper/user.go b/scraper/user.go index 1834cd2..1526c5e 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -23,31 +23,31 @@ func JoinArrayOfHandles(handles []UserHandle) string { } type User struct { - ID UserID - DisplayName string - Handle UserHandle - Bio string - FollowingCount int - FollowersCount int - Location string - Website string - JoinDate Timestamp - IsPrivate bool - IsVerified bool - IsBanned bool - IsDeleted bool - ProfileImageUrl string - ProfileImageLocalPath string - BannerImageUrl string - BannerImageLocalPath string + ID UserID `db:"id"` + DisplayName string `db:"display_name"` + Handle UserHandle `db:"handle"` + Bio string `db:"bio"` + FollowingCount int `db:"following_count"` + FollowersCount int `db:"followers_count"` + Location string `db:"location"` + Website string `db:"website"` + JoinDate Timestamp `db:"join_date"` + IsPrivate bool `db:"is_private"` + IsVerified bool `db:"is_verified"` + IsBanned bool `db:"is_banned"` + IsDeleted bool `db:"is_deleted"` + ProfileImageUrl string `db:"profile_image_url"` + ProfileImageLocalPath string `db:"profile_image_local_path"` + BannerImageUrl string `db:"banner_image_url"` + BannerImageLocalPath string `db:"banner_image_local_path"` - PinnedTweetID TweetID + PinnedTweetID TweetID `db:"pinned_tweet_id"` PinnedTweet *Tweet - IsFollowed bool - IsContentDownloaded bool + IsFollowed bool `db:"is_followed"` + IsContentDownloaded bool `db:"is_content_downloaded"` IsNeedingFakeID bool - IsIdFake bool + IsIdFake bool `db:"is_id_fake"` } func (u User) String() string { diff --git a/scraper/video.go b/scraper/video.go index 0288c64..a3c3ae5 100644 --- a/scraper/video.go +++ b/scraper/video.go @@ -12,21 +12,21 @@ type VideoID int64 // from someone else). type Video struct { - ID VideoID - TweetID TweetID - Width int - Height int - RemoteURL string - LocalFilename string + ID VideoID `db:"id"` + TweetID TweetID `db:"tweet_id"` + Width int `db:"width"` + Height int `db:"height"` + RemoteURL string `db:"remote_url"` + LocalFilename string `db:"local_filename"` - ThumbnailRemoteUrl string + ThumbnailRemoteUrl string `db:"thumbnail_remote_url"` ThumbnailLocalPath string `db:"thumbnail_local_filename"` - Duration int // milliseconds - ViewCount int + Duration int `db:"duration"` // milliseconds + ViewCount int `db:"view_count"` - IsDownloaded bool - IsBlockedByDMCA bool - IsGif bool + IsDownloaded bool `db:"is_downloaded"` + IsBlockedByDMCA bool `db:"is_blocked_by_dmca"` + IsGif bool `db:"is_gif"` } func get_filename(remote_url string) string {