REFACTOR: change all appropriate queries to use sqlx
This commit is contained in:
parent
6e5e8dba67
commit
cae95ed99d
@ -13,14 +13,13 @@ import (
|
|||||||
* - img: the Image to save
|
* - img: the Image to save
|
||||||
*/
|
*/
|
||||||
func (p Profile) SaveImage(img scraper.Image) error {
|
func (p Profile) SaveImage(img scraper.Image) error {
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded)
|
insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded)
|
||||||
values (?, ?, ?, ?, ?, ?, ?)
|
values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :is_downloaded)
|
||||||
on conflict do update
|
on conflict do update
|
||||||
set is_downloaded=(is_downloaded or ?)
|
set is_downloaded=(is_downloaded or :is_downloaded)
|
||||||
`,
|
`,
|
||||||
img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
|
img,
|
||||||
img.IsDownloaded,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error saving image (tweet ID %d):\n %w", img.TweetID, err)
|
return fmt.Errorf("Error saving image (tweet ID %d):\n %w", img.TweetID, err)
|
||||||
@ -35,19 +34,17 @@ func (p Profile) SaveImage(img scraper.Image) error {
|
|||||||
* - img: the Video to save
|
* - img: the Video to save
|
||||||
*/
|
*/
|
||||||
func (p Profile) SaveVideo(vid scraper.Video) error {
|
func (p Profile) SaveVideo(vid scraper.Video) error {
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
|
insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
|
||||||
duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
|
duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url, :thumbnail_local_filename,
|
||||||
|
:duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif)
|
||||||
on conflict do update
|
on conflict do update
|
||||||
set is_downloaded=(is_downloaded or ?),
|
set is_downloaded=(is_downloaded or :is_downloaded),
|
||||||
view_count=max(view_count, ?),
|
view_count=max(view_count, :view_count),
|
||||||
is_blocked_by_dmca = ?
|
is_blocked_by_dmca = :is_blocked_by_dmca
|
||||||
`,
|
`,
|
||||||
vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath,
|
vid,
|
||||||
vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsBlockedByDMCA, vid.IsGif,
|
|
||||||
|
|
||||||
vid.IsDownloaded, vid.ViewCount, vid.IsBlockedByDMCA,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err)
|
return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err)
|
||||||
@ -59,17 +56,16 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
|
|||||||
* Save an Url
|
* Save an Url
|
||||||
*/
|
*/
|
||||||
func (p Profile) SaveUrl(url scraper.Url) error {
|
func (p Profile) SaveUrl(url scraper.Url) error {
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
|
insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
|
||||||
thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
|
thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
values (:tweet_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width,
|
||||||
|
:thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded
|
||||||
|
)
|
||||||
on conflict do update
|
on conflict do update
|
||||||
set is_content_downloaded=(is_content_downloaded or ?)
|
set is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
|
||||||
`,
|
`,
|
||||||
url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth,
|
url,
|
||||||
url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
|
|
||||||
|
|
||||||
url.IsContentDownloaded,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error saving Url (tweet ID %d):\n %w", url.TweetID, err)
|
return fmt.Errorf("Error saving Url (tweet ID %d):\n %w", url.TweetID, err)
|
||||||
@ -81,21 +77,19 @@ func (p Profile) SaveUrl(url scraper.Url) error {
|
|||||||
* Save a Poll
|
* Save a Poll
|
||||||
*/
|
*/
|
||||||
func (p Profile) SavePoll(poll scraper.Poll) error {
|
func (p Profile) SavePoll(poll scraper.Poll) error {
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4,
|
insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4,
|
||||||
choice4_votes, voting_duration, voting_ends_at, last_scraped_at)
|
choice4_votes, voting_duration, voting_ends_at, last_scraped_at)
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
values (:id, :tweet_id, :num_choices, :choice1, :choice1_votes, :choice2, :choice2_votes, :choice3, :choice3_votes,
|
||||||
|
:choice4, :choice4_votes, :voting_duration, :voting_ends_at, :last_scraped_at)
|
||||||
on conflict do update
|
on conflict do update
|
||||||
set choice1_votes=?,
|
set choice1_votes=:choice1_votes,
|
||||||
choice2_votes=?,
|
choice2_votes=:choice2_votes,
|
||||||
choice3_votes=?,
|
choice3_votes=:choice3_votes,
|
||||||
choice4_votes=?,
|
choice4_votes=:choice4_votes,
|
||||||
last_scraped_at=?
|
last_scraped_at=:last_scraped_at
|
||||||
`,
|
`,
|
||||||
poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3,
|
poll,
|
||||||
poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt, poll.LastUpdatedAt,
|
|
||||||
|
|
||||||
poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error saving Poll (tweet ID %d):\n %w", poll.TweetID, err)
|
return fmt.Errorf("Error saving Poll (tweet ID %d):\n %w", poll.TweetID, err)
|
||||||
|
@ -10,12 +10,12 @@ import (
|
|||||||
* Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable.
|
* Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable.
|
||||||
*/
|
*/
|
||||||
func (p Profile) SaveRetweet(r scraper.Retweet) error {
|
func (p Profile) SaveRetweet(r scraper.Retweet) error {
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at)
|
insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at)
|
||||||
values (?, ?, ?, ?)
|
values (:retweet_id, :tweet_id, :retweeted_by, :retweeted_at)
|
||||||
on conflict do nothing
|
on conflict do nothing
|
||||||
`,
|
`,
|
||||||
r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(),
|
r,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error executing SaveRetweet(%#v):\n %w", r, err)
|
return fmt.Errorf("Error executing SaveRetweet(%#v):\n %w", r, err)
|
||||||
|
@ -33,35 +33,32 @@ func (p Profile) SaveUser(u *scraper.User) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := p.DB.Exec(`
|
_, err := p.DB.NamedExec(`
|
||||||
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private,
|
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private,
|
||||||
is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
|
is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
|
||||||
pinned_tweet_id, is_content_downloaded, is_id_fake)
|
pinned_tweet_id, is_content_downloaded, is_id_fake)
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
values (:id, :display_name, :handle, :bio, :following_count, :followers_count, :location, :website, :join_date, :is_private,
|
||||||
|
:is_verified, :is_banned, :profile_image_url, :profile_image_local_path, :banner_image_url, :banner_image_local_path,
|
||||||
|
:pinned_tweet_id, :is_content_downloaded, :is_id_fake)
|
||||||
on conflict do update
|
on conflict do update
|
||||||
set handle=?,
|
set handle=:handle,
|
||||||
bio=?,
|
bio=:bio,
|
||||||
display_name=?,
|
display_name=:display_name,
|
||||||
following_count=?,
|
following_count=:following_count,
|
||||||
followers_count=?,
|
followers_count=:followers_count,
|
||||||
location=?,
|
location=:location,
|
||||||
website=?,
|
website=:website,
|
||||||
is_private=?,
|
is_private=:is_private,
|
||||||
is_verified=?,
|
is_verified=:is_verified,
|
||||||
is_banned=?,
|
is_banned=:is_banned,
|
||||||
profile_image_url=?,
|
profile_image_url=:profile_image_url,
|
||||||
profile_image_local_path=?,
|
profile_image_local_path=:profile_image_local_path,
|
||||||
banner_image_url=?,
|
banner_image_url=:banner_image_url,
|
||||||
banner_image_local_path=?,
|
banner_image_local_path=:banner_image_local_path,
|
||||||
pinned_tweet_id=?,
|
pinned_tweet_id=:pinned_tweet_id,
|
||||||
is_content_downloaded=(is_content_downloaded or ?)
|
is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
|
||||||
`,
|
`,
|
||||||
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate, u.IsPrivate,
|
u,
|
||||||
u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
|
|
||||||
u.IsContentDownloaded, u.IsIdFake,
|
|
||||||
|
|
||||||
u.Handle, u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned,
|
|
||||||
u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error executing SaveUser(%s):\n %w", u.Handle, err)
|
return fmt.Errorf("Error executing SaveUser(%s):\n %w", u.Handle, err)
|
||||||
|
@ -7,13 +7,13 @@ import (
|
|||||||
type ImageID int64
|
type ImageID int64
|
||||||
|
|
||||||
type Image struct {
|
type Image struct {
|
||||||
ID ImageID
|
ID ImageID `db:"id"`
|
||||||
TweetID TweetID
|
TweetID TweetID `db:"tweet_id"`
|
||||||
Width int
|
Width int `db:"width"`
|
||||||
Height int
|
Height int `db:"height"`
|
||||||
RemoteURL string
|
RemoteURL string `db:"remote_url"`
|
||||||
LocalFilename string
|
LocalFilename string `db:"local_filename"`
|
||||||
IsDownloaded bool
|
IsDownloaded bool `db:"is_downloaded"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseAPIMedia(apiMedia APIMedia) Image {
|
func ParseAPIMedia(apiMedia APIMedia) Image {
|
||||||
|
@ -9,21 +9,21 @@ import (
|
|||||||
type PollID int64
|
type PollID int64
|
||||||
|
|
||||||
type Poll struct {
|
type Poll struct {
|
||||||
ID PollID
|
ID PollID `db:"id"`
|
||||||
TweetID TweetID
|
TweetID TweetID `db:"tweet_id"`
|
||||||
NumChoices int
|
NumChoices int `db:"num_choices"`
|
||||||
|
|
||||||
Choice1 string
|
Choice1 string `db:"choice1"`
|
||||||
Choice1_Votes int
|
Choice1_Votes int `db:"choice1_votes"`
|
||||||
Choice2 string
|
Choice2 string `db:"choice2"`
|
||||||
Choice2_Votes int
|
Choice2_Votes int `db:"choice2_votes"`
|
||||||
Choice3 string
|
Choice3 string `db:"choice3"`
|
||||||
Choice3_Votes int
|
Choice3_Votes int `db:"choice3_votes"`
|
||||||
Choice4 string
|
Choice4 string `db:"choice4"`
|
||||||
Choice4_Votes int
|
Choice4_Votes int `db:"choice4_votes"`
|
||||||
|
|
||||||
VotingDuration int // In seconds
|
VotingDuration int `db:"voting_duration"` // In seconds
|
||||||
VotingEndsAt Timestamp
|
VotingEndsAt Timestamp `db:"voting_ends_at"`
|
||||||
|
|
||||||
LastUpdatedAt Timestamp `db:"last_scraped_at"`
|
LastUpdatedAt Timestamp `db:"last_scraped_at"`
|
||||||
}
|
}
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
package scraper
|
package scraper
|
||||||
|
|
||||||
type Retweet struct {
|
type Retweet struct {
|
||||||
RetweetID TweetID
|
RetweetID TweetID `db:"retweet_id"`
|
||||||
TweetID TweetID
|
TweetID TweetID `db:"tweet_id"`
|
||||||
Tweet *Tweet
|
Tweet *Tweet
|
||||||
RetweetedByID UserID `db:"retweeted_by"`
|
RetweetedByID UserID `db:"retweeted_by"`
|
||||||
RetweetedBy *User
|
RetweetedBy *User
|
||||||
RetweetedAt Timestamp
|
RetweetedAt Timestamp `db:"retweeted_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) {
|
func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) {
|
||||||
|
@ -8,23 +8,22 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Url struct {
|
type Url struct {
|
||||||
TweetID TweetID
|
TweetID TweetID `db:"tweet_id"`
|
||||||
|
Domain string `db:"domain"`
|
||||||
|
Text string `db:"text"`
|
||||||
|
ShortText string `db:"short_text"`
|
||||||
|
Title string `db:"title"`
|
||||||
|
Description string `db:"description"`
|
||||||
|
ThumbnailWidth int `db:"thumbnail_width"`
|
||||||
|
ThumbnailHeight int `db:"thumbnail_height"`
|
||||||
|
ThumbnailRemoteUrl string `db:"thumbnail_remote_url"`
|
||||||
|
ThumbnailLocalPath string `db:"thumbnail_local_path"`
|
||||||
|
CreatorID UserID `db:"creator_id"`
|
||||||
|
SiteID UserID `db:"site_id"`
|
||||||
|
|
||||||
Domain string
|
HasCard bool `db:"has_card"`
|
||||||
Text string
|
HasThumbnail bool `db:"has_thumbnail"`
|
||||||
ShortText string
|
IsContentDownloaded bool `db:"is_content_downloaded"`
|
||||||
Title string
|
|
||||||
Description string
|
|
||||||
ThumbnailWidth int
|
|
||||||
ThumbnailHeight int
|
|
||||||
ThumbnailRemoteUrl string
|
|
||||||
ThumbnailLocalPath string
|
|
||||||
CreatorID UserID
|
|
||||||
SiteID UserID
|
|
||||||
|
|
||||||
HasCard bool
|
|
||||||
HasThumbnail bool
|
|
||||||
IsContentDownloaded bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseAPIUrlCard(apiCard APICard) Url {
|
func ParseAPIUrlCard(apiCard APICard) Url {
|
||||||
|
@ -23,31 +23,31 @@ func JoinArrayOfHandles(handles []UserHandle) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type User struct {
|
type User struct {
|
||||||
ID UserID
|
ID UserID `db:"id"`
|
||||||
DisplayName string
|
DisplayName string `db:"display_name"`
|
||||||
Handle UserHandle
|
Handle UserHandle `db:"handle"`
|
||||||
Bio string
|
Bio string `db:"bio"`
|
||||||
FollowingCount int
|
FollowingCount int `db:"following_count"`
|
||||||
FollowersCount int
|
FollowersCount int `db:"followers_count"`
|
||||||
Location string
|
Location string `db:"location"`
|
||||||
Website string
|
Website string `db:"website"`
|
||||||
JoinDate Timestamp
|
JoinDate Timestamp `db:"join_date"`
|
||||||
IsPrivate bool
|
IsPrivate bool `db:"is_private"`
|
||||||
IsVerified bool
|
IsVerified bool `db:"is_verified"`
|
||||||
IsBanned bool
|
IsBanned bool `db:"is_banned"`
|
||||||
IsDeleted bool
|
IsDeleted bool `db:"is_deleted"`
|
||||||
ProfileImageUrl string
|
ProfileImageUrl string `db:"profile_image_url"`
|
||||||
ProfileImageLocalPath string
|
ProfileImageLocalPath string `db:"profile_image_local_path"`
|
||||||
BannerImageUrl string
|
BannerImageUrl string `db:"banner_image_url"`
|
||||||
BannerImageLocalPath string
|
BannerImageLocalPath string `db:"banner_image_local_path"`
|
||||||
|
|
||||||
PinnedTweetID TweetID
|
PinnedTweetID TweetID `db:"pinned_tweet_id"`
|
||||||
PinnedTweet *Tweet
|
PinnedTweet *Tweet
|
||||||
|
|
||||||
IsFollowed bool
|
IsFollowed bool `db:"is_followed"`
|
||||||
IsContentDownloaded bool
|
IsContentDownloaded bool `db:"is_content_downloaded"`
|
||||||
IsNeedingFakeID bool
|
IsNeedingFakeID bool
|
||||||
IsIdFake bool
|
IsIdFake bool `db:"is_id_fake"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u User) String() string {
|
func (u User) String() string {
|
||||||
|
@ -12,21 +12,21 @@ type VideoID int64
|
|||||||
// from someone else).
|
// from someone else).
|
||||||
|
|
||||||
type Video struct {
|
type Video struct {
|
||||||
ID VideoID
|
ID VideoID `db:"id"`
|
||||||
TweetID TweetID
|
TweetID TweetID `db:"tweet_id"`
|
||||||
Width int
|
Width int `db:"width"`
|
||||||
Height int
|
Height int `db:"height"`
|
||||||
RemoteURL string
|
RemoteURL string `db:"remote_url"`
|
||||||
LocalFilename string
|
LocalFilename string `db:"local_filename"`
|
||||||
|
|
||||||
ThumbnailRemoteUrl string
|
ThumbnailRemoteUrl string `db:"thumbnail_remote_url"`
|
||||||
ThumbnailLocalPath string `db:"thumbnail_local_filename"`
|
ThumbnailLocalPath string `db:"thumbnail_local_filename"`
|
||||||
Duration int // milliseconds
|
Duration int `db:"duration"` // milliseconds
|
||||||
ViewCount int
|
ViewCount int `db:"view_count"`
|
||||||
|
|
||||||
IsDownloaded bool
|
IsDownloaded bool `db:"is_downloaded"`
|
||||||
IsBlockedByDMCA bool
|
IsBlockedByDMCA bool `db:"is_blocked_by_dmca"`
|
||||||
IsGif bool
|
IsGif bool `db:"is_gif"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func get_filename(remote_url string) string {
|
func get_filename(remote_url string) string {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user