REFACTOR: change all appropriate queries to use sqlx

persistence/media_queries.go

@@ -13,14 +13,13 @@ import (
  * - img: the Image to save
  */
 func (p Profile) SaveImage(img scraper.Image) error {
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
 		insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded)
-		            values (?, ?, ?, ?, ?, ?, ?)
+		            values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :is_downloaded)
 		       on conflict do update
-		               set is_downloaded=(is_downloaded or ?)
+		               set is_downloaded=(is_downloaded or :is_downloaded)
 		`,
-		img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
+		img,
-		img.IsDownloaded,
 	)
 	if err != nil {
 		return fmt.Errorf("Error saving image (tweet ID %d):\n  %w", img.TweetID, err)
@@ -35,19 +34,17 @@ func (p Profile) SaveImage(img scraper.Image) error {
  * - img: the Video to save
  */
 func (p Profile) SaveVideo(vid scraper.Video) error {
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
 		insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
 		                    duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
-		            values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		            values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url, :thumbnail_local_filename,
+                            :duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif)
 		       on conflict do update
-		               set is_downloaded=(is_downloaded or ?),
+		               set is_downloaded=(is_downloaded or :is_downloaded),
-		                   view_count=max(view_count, ?),
+		                   view_count=max(view_count, :view_count),
-						   is_blocked_by_dmca = ?
+						   is_blocked_by_dmca = :is_blocked_by_dmca
 		`,
-		vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath,
+		vid,
-		vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsBlockedByDMCA, vid.IsGif,
-
-		vid.IsDownloaded, vid.ViewCount, vid.IsBlockedByDMCA,
 	)
 	if err != nil {
 		return fmt.Errorf("Error saving video (tweet ID %d):\n  %w", vid.TweetID, err)
@@ -59,17 +56,16 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
  * Save an Url
  */
 func (p Profile) SaveUrl(url scraper.Url) error {
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
 		insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
 		                  thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
-		          values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		          values (:tweet_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width,
+                          :thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded
+                         )
 		     on conflict do update
-		             set is_content_downloaded=(is_content_downloaded or ?)
+		             set is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
 		`,
-		url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth,
+		url,
-		url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
-
-		url.IsContentDownloaded,
 	)
 	if err != nil {
 		return fmt.Errorf("Error saving Url (tweet ID %d):\n  %w", url.TweetID, err)
@@ -81,21 +77,19 @@ func (p Profile) SaveUrl(url scraper.Url) error {
  * Save a Poll
  */
 func (p Profile) SavePoll(poll scraper.Poll) error {
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
 		insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4,
 		                   choice4_votes, voting_duration, voting_ends_at, last_scraped_at)
-		           values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		           values (:id, :tweet_id, :num_choices, :choice1, :choice1_votes, :choice2, :choice2_votes, :choice3, :choice3_votes,
+                           :choice4, :choice4_votes, :voting_duration, :voting_ends_at, :last_scraped_at)
 		      on conflict do update
-		              set choice1_votes=?,
+		              set choice1_votes=:choice1_votes,
-		                  choice2_votes=?,
+		                  choice2_votes=:choice2_votes,
-		                  choice3_votes=?,
+		                  choice3_votes=:choice3_votes,
-		                  choice4_votes=?,
+		                  choice4_votes=:choice4_votes,
-		                  last_scraped_at=?
+		                  last_scraped_at=:last_scraped_at
 		`,
-		poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3,
+		poll,
-		poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt, poll.LastUpdatedAt,
-
-		poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt,
 	)
 	if err != nil {
 		return fmt.Errorf("Error saving Poll (tweet ID %d):\n  %w", poll.TweetID, err)

persistence/retweet_queries.go

@@ -10,12 +10,12 @@ import (
  * Save a Retweet.  Do nothing if it already exists, because none of its parameters are modifiable.
  */
 func (p Profile) SaveRetweet(r scraper.Retweet) error {
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
 			insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at)
-			values (?, ?, ?, ?)
+			values (:retweet_id, :tweet_id, :retweeted_by, :retweeted_at)
 			    on conflict do nothing
 		`,
-		r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(),
+		r,
 	)
 	if err != nil {
 		return fmt.Errorf("Error executing SaveRetweet(%#v):\n  %w", r, err)

persistence/user_queries.go

@@ -33,35 +33,32 @@ func (p Profile) SaveUser(u *scraper.User) error {
 		}
 	}
 
-	_, err := p.DB.Exec(`
+	_, err := p.DB.NamedExec(`
         insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private,
                            is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
                            pinned_tweet_id, is_content_downloaded, is_id_fake)
-        values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        values (:id, :display_name, :handle, :bio, :following_count, :followers_count, :location, :website, :join_date, :is_private,
+                :is_verified, :is_banned, :profile_image_url, :profile_image_local_path, :banner_image_url, :banner_image_local_path,
+                :pinned_tweet_id, :is_content_downloaded, :is_id_fake)
             on conflict do update
-           set handle=?,
+           set handle=:handle,
-               bio=?,
+               bio=:bio,
-               display_name=?,
+               display_name=:display_name,
-               following_count=?,
+               following_count=:following_count,
-               followers_count=?,
+               followers_count=:followers_count,
-               location=?,
+               location=:location,
-               website=?,
+               website=:website,
-               is_private=?,
+               is_private=:is_private,
-               is_verified=?,
+               is_verified=:is_verified,
-               is_banned=?,
+               is_banned=:is_banned,
-               profile_image_url=?,
+               profile_image_url=:profile_image_url,
-               profile_image_local_path=?,
+               profile_image_local_path=:profile_image_local_path,
-               banner_image_url=?,
+               banner_image_url=:banner_image_url,
-               banner_image_local_path=?,
+               banner_image_local_path=:banner_image_local_path,
-               pinned_tweet_id=?,
+               pinned_tweet_id=:pinned_tweet_id,
-               is_content_downloaded=(is_content_downloaded or ?)
+               is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
         `,
-		u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate, u.IsPrivate,
+		u,
-		u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
-		u.IsContentDownloaded, u.IsIdFake,
-
-		u.Handle, u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned,
-		u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
 	)
 	if err != nil {
 		return fmt.Errorf("Error executing SaveUser(%s):\n  %w", u.Handle, err)

scraper/image.go

@@ -7,13 +7,13 @@ import (
 type ImageID int64
 
 type Image struct {
-	ID            ImageID
+	ID            ImageID `db:"id"`
-	TweetID       TweetID
+	TweetID       TweetID `db:"tweet_id"`
-	Width         int
+	Width         int     `db:"width"`
-	Height        int
+	Height        int     `db:"height"`
-	RemoteURL     string
+	RemoteURL     string  `db:"remote_url"`
-	LocalFilename string
+	LocalFilename string  `db:"local_filename"`
-	IsDownloaded  bool
+	IsDownloaded  bool    `db:"is_downloaded"`
 }
 
 func ParseAPIMedia(apiMedia APIMedia) Image {

scraper/poll.go

@@ -9,21 +9,21 @@ import (
 type PollID int64
 
 type Poll struct {
-	ID         PollID
+	ID         PollID  `db:"id"`
-	TweetID    TweetID
+	TweetID    TweetID `db:"tweet_id"`
-	NumChoices int
+	NumChoices int     `db:"num_choices"`
 
-	Choice1       string
+	Choice1       string `db:"choice1"`
-	Choice1_Votes int
+	Choice1_Votes int    `db:"choice1_votes"`
-	Choice2       string
+	Choice2       string `db:"choice2"`
-	Choice2_Votes int
+	Choice2_Votes int    `db:"choice2_votes"`
-	Choice3       string
+	Choice3       string `db:"choice3"`
-	Choice3_Votes int
+	Choice3_Votes int    `db:"choice3_votes"`
-	Choice4       string
+	Choice4       string `db:"choice4"`
-	Choice4_Votes int
+	Choice4_Votes int    `db:"choice4_votes"`
 
-	VotingDuration int // In seconds
+	VotingDuration int       `db:"voting_duration"` // In seconds
-	VotingEndsAt   Timestamp
+	VotingEndsAt   Timestamp `db:"voting_ends_at"`
 
 	LastUpdatedAt Timestamp `db:"last_scraped_at"`
 }

scraper/retweet.go

@@ -1,12 +1,12 @@
 package scraper
 
 type Retweet struct {
-	RetweetID     TweetID
+	RetweetID     TweetID `db:"retweet_id"`
-	TweetID       TweetID
+	TweetID       TweetID `db:"tweet_id"`
 	Tweet         *Tweet
 	RetweetedByID UserID `db:"retweeted_by"`
 	RetweetedBy   *User
-	RetweetedAt   Timestamp
+	RetweetedAt   Timestamp `db:"retweeted_at"`
 }
 
 func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) {

scraper/url.go

@@ -8,23 +8,22 @@ import (
 )
 
 type Url struct {
-	TweetID TweetID
+	TweetID            TweetID `db:"tweet_id"`
+	Domain             string  `db:"domain"`
+	Text               string  `db:"text"`
+	ShortText          string  `db:"short_text"`
+	Title              string  `db:"title"`
+	Description        string  `db:"description"`
+	ThumbnailWidth     int     `db:"thumbnail_width"`
+	ThumbnailHeight    int     `db:"thumbnail_height"`
+	ThumbnailRemoteUrl string  `db:"thumbnail_remote_url"`
+	ThumbnailLocalPath string  `db:"thumbnail_local_path"`
+	CreatorID          UserID  `db:"creator_id"`
+	SiteID             UserID  `db:"site_id"`
 
-	Domain             string
+	HasCard             bool `db:"has_card"`
-	Text               string
+	HasThumbnail        bool `db:"has_thumbnail"`
-	ShortText          string
+	IsContentDownloaded bool `db:"is_content_downloaded"`
-	Title              string
-	Description        string
-	ThumbnailWidth     int
-	ThumbnailHeight    int
-	ThumbnailRemoteUrl string
-	ThumbnailLocalPath string
-	CreatorID          UserID
-	SiteID             UserID
-
-	HasCard             bool
-	HasThumbnail        bool
-	IsContentDownloaded bool
 }
 
 func ParseAPIUrlCard(apiCard APICard) Url {

scraper/user.go

@@ -23,31 +23,31 @@ func JoinArrayOfHandles(handles []UserHandle) string {
 }
 
 type User struct {
-	ID                    UserID
+	ID                    UserID     `db:"id"`
-	DisplayName           string
+	DisplayName           string     `db:"display_name"`
-	Handle                UserHandle
+	Handle                UserHandle `db:"handle"`
-	Bio                   string
+	Bio                   string     `db:"bio"`
-	FollowingCount        int
+	FollowingCount        int        `db:"following_count"`
-	FollowersCount        int
+	FollowersCount        int        `db:"followers_count"`
-	Location              string
+	Location              string     `db:"location"`
-	Website               string
+	Website               string     `db:"website"`
-	JoinDate              Timestamp
+	JoinDate              Timestamp  `db:"join_date"`
-	IsPrivate             bool
+	IsPrivate             bool       `db:"is_private"`
-	IsVerified            bool
+	IsVerified            bool       `db:"is_verified"`
-	IsBanned              bool
+	IsBanned              bool       `db:"is_banned"`
-	IsDeleted             bool
+	IsDeleted             bool       `db:"is_deleted"`
-	ProfileImageUrl       string
+	ProfileImageUrl       string     `db:"profile_image_url"`
-	ProfileImageLocalPath string
+	ProfileImageLocalPath string     `db:"profile_image_local_path"`
-	BannerImageUrl        string
+	BannerImageUrl        string     `db:"banner_image_url"`
-	BannerImageLocalPath  string
+	BannerImageLocalPath  string     `db:"banner_image_local_path"`
 
-	PinnedTweetID TweetID
+	PinnedTweetID TweetID `db:"pinned_tweet_id"`
 	PinnedTweet   *Tweet
 
-	IsFollowed          bool
+	IsFollowed          bool `db:"is_followed"`
-	IsContentDownloaded bool
+	IsContentDownloaded bool `db:"is_content_downloaded"`
 	IsNeedingFakeID     bool
-	IsIdFake            bool
+	IsIdFake            bool `db:"is_id_fake"`
 }
 
 func (u User) String() string {

scraper/video.go

@@ -12,21 +12,21 @@ type VideoID int64
 // from someone else).
 
 type Video struct {
-	ID            VideoID
+	ID            VideoID `db:"id"`
-	TweetID       TweetID
+	TweetID       TweetID `db:"tweet_id"`
-	Width         int
+	Width         int     `db:"width"`
-	Height        int
+	Height        int     `db:"height"`
-	RemoteURL     string
+	RemoteURL     string  `db:"remote_url"`
-	LocalFilename string
+	LocalFilename string  `db:"local_filename"`
 
-	ThumbnailRemoteUrl string
+	ThumbnailRemoteUrl string `db:"thumbnail_remote_url"`
 	ThumbnailLocalPath string `db:"thumbnail_local_filename"`
-	Duration           int    // milliseconds
+	Duration           int    `db:"duration"` // milliseconds
-	ViewCount          int
+	ViewCount          int    `db:"view_count"`
 
-	IsDownloaded    bool
+	IsDownloaded    bool `db:"is_downloaded"`
-	IsBlockedByDMCA bool
+	IsBlockedByDMCA bool `db:"is_blocked_by_dmca"`
-	IsGif           bool
+	IsGif           bool `db:"is_gif"`
 }
 
 func get_filename(remote_url string) string {