me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: change all appropriate queries to use sqlx

Browse Source
This commit is contained in:
Alessio 2023-06-27 21:56:29 -03:00
parent 6e5e8dba67
commit cae95ed99d
9 changed files with 121 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
persistence/media_queries.go
View File

@ -13,14 +13,13 @@ import (
* - img: the Image to save
*/
func (p Profile) SaveImage(img scraper.Image) error {
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?, ?, ?)
values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :is_downloaded)
on conflict do update
set is_downloaded=(is_downloaded or ?)
set is_downloaded=(is_downloaded or :is_downloaded)
`,
img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
img.IsDownloaded,
img,
)
if err != nil {
return fmt.Errorf("Error saving image (tweet ID %d):\n %w", img.TweetID, err)
@ -35,19 +34,17 @@ func (p Profile) SaveImage(img scraper.Image) error {
* - img: the Video to save
*/
func (p Profile) SaveVideo(vid scraper.Video) error {
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
values (:id, :tweet_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url, :thumbnail_local_filename,
:duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif)
on conflict do update
set is_downloaded=(is_downloaded or ?),
view_count=max(view_count, ?),
is_blocked_by_dmca = ?
set is_downloaded=(is_downloaded or :is_downloaded),
view_count=max(view_count, :view_count),
is_blocked_by_dmca = :is_blocked_by_dmca
`,
vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath,
vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsBlockedByDMCA, vid.IsGif,
vid.IsDownloaded, vid.ViewCount, vid.IsBlockedByDMCA,
vid,
)
if err != nil {
return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err)
@ -59,17 +56,16 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
* Save an Url
*/
func (p Profile) SaveUrl(url scraper.Url) error {
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
values (:tweet_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width,
:thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded
)
on conflict do update
set is_content_downloaded=(is_content_downloaded or ?)
set is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
`,
url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth,
url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
url.IsContentDownloaded,
url,
)
if err != nil {
return fmt.Errorf("Error saving Url (tweet ID %d):\n %w", url.TweetID, err)
@ -81,21 +77,19 @@ func (p Profile) SaveUrl(url scraper.Url) error {
* Save a Poll
*/
func (p Profile) SavePoll(poll scraper.Poll) error {
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4,
choice4_votes, voting_duration, voting_ends_at, last_scraped_at)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
values (:id, :tweet_id, :num_choices, :choice1, :choice1_votes, :choice2, :choice2_votes, :choice3, :choice3_votes,
:choice4, :choice4_votes, :voting_duration, :voting_ends_at, :last_scraped_at)
on conflict do update
set choice1_votes=?,
choice2_votes=?,
choice3_votes=?,
choice4_votes=?,
last_scraped_at=?
set choice1_votes=:choice1_votes,
choice2_votes=:choice2_votes,
choice3_votes=:choice3_votes,
choice4_votes=:choice4_votes,
last_scraped_at=:last_scraped_at
`,
poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3,
poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt, poll.LastUpdatedAt,
poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt,
poll,
)
if err != nil {
return fmt.Errorf("Error saving Poll (tweet ID %d):\n %w", poll.TweetID, err)

6
persistence/retweet_queries.go
View File

@ -10,12 +10,12 @@ import (
* Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable.
*/
func (p Profile) SaveRetweet(r scraper.Retweet) error {
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into retweets (retweet_id, tweet_id, retweeted_by, retweeted_at)
values (?, ?, ?, ?)
values (:retweet_id, :tweet_id, :retweeted_by, :retweeted_at)
on conflict do nothing
`,
r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(),
r,
)
if err != nil {
return fmt.Errorf("Error executing SaveRetweet(%#v):\n %w", r, err)

45
persistence/user_queries.go
View File

@ -33,35 +33,32 @@ func (p Profile) SaveUser(u *scraper.User) error {
}
}
_, err := p.DB.Exec(`
_, err := p.DB.NamedExec(`
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private,
is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
pinned_tweet_id, is_content_downloaded, is_id_fake)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
values (:id, :display_name, :handle, :bio, :following_count, :followers_count, :location, :website, :join_date, :is_private,
:is_verified, :is_banned, :profile_image_url, :profile_image_local_path, :banner_image_url, :banner_image_local_path,
:pinned_tweet_id, :is_content_downloaded, :is_id_fake)
on conflict do update
set handle=?,
bio=?,
display_name=?,
following_count=?,
followers_count=?,
location=?,
website=?,
is_private=?,
is_verified=?,
is_banned=?,
profile_image_url=?,
profile_image_local_path=?,
banner_image_url=?,
banner_image_local_path=?,
pinned_tweet_id=?,
is_content_downloaded=(is_content_downloaded or ?)
set handle=:handle,
bio=:bio,
display_name=:display_name,
following_count=:following_count,
followers_count=:followers_count,
location=:location,
website=:website,
is_private=:is_private,
is_verified=:is_verified,
is_banned=:is_banned,
profile_image_url=:profile_image_url,
profile_image_local_path=:profile_image_local_path,
banner_image_url=:banner_image_url,
banner_image_local_path=:banner_image_local_path,
pinned_tweet_id=:pinned_tweet_id,
is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
`,
u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate, u.IsPrivate,
u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID,
u.IsContentDownloaded, u.IsIdFake,
u.Handle, u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned,
u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
u,
)
if err != nil {
return fmt.Errorf("Error executing SaveUser(%s):\n %w", u.Handle, err)

14
scraper/image.go
View File

@ -7,13 +7,13 @@ import (
type ImageID int64
type Image struct {
ID ImageID
TweetID TweetID
Width int
Height int
RemoteURL string
LocalFilename string
IsDownloaded bool
ID ImageID `db:"id"`
TweetID TweetID `db:"tweet_id"`
Width int `db:"width"`
Height int `db:"height"`
RemoteURL string `db:"remote_url"`
LocalFilename string `db:"local_filename"`
IsDownloaded bool `db:"is_downloaded"`
}
func ParseAPIMedia(apiMedia APIMedia) Image {

26
scraper/poll.go
View File

@ -9,21 +9,21 @@ import (
type PollID int64
type Poll struct {
ID PollID
TweetID TweetID
NumChoices int
ID PollID `db:"id"`
TweetID TweetID `db:"tweet_id"`
NumChoices int `db:"num_choices"`
Choice1 string
Choice1_Votes int
Choice2 string
Choice2_Votes int
Choice3 string
Choice3_Votes int
Choice4 string
Choice4_Votes int
Choice1 string `db:"choice1"`
Choice1_Votes int `db:"choice1_votes"`
Choice2 string `db:"choice2"`
Choice2_Votes int `db:"choice2_votes"`
Choice3 string `db:"choice3"`
Choice3_Votes int `db:"choice3_votes"`
Choice4 string `db:"choice4"`
Choice4_Votes int `db:"choice4_votes"`
VotingDuration int // In seconds
VotingEndsAt Timestamp
VotingDuration int `db:"voting_duration"` // In seconds
VotingEndsAt Timestamp `db:"voting_ends_at"`
LastUpdatedAt Timestamp `db:"last_scraped_at"`
}

6
scraper/retweet.go
View File

@ -1,12 +1,12 @@
package scraper
type Retweet struct {
RetweetID TweetID
TweetID TweetID
RetweetID TweetID `db:"retweet_id"`
TweetID TweetID `db:"tweet_id"`
Tweet *Tweet
RetweetedByID UserID `db:"retweeted_by"`
RetweetedBy *User
RetweetedAt Timestamp
RetweetedAt Timestamp `db:"retweeted_at"`
}
func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) {

31
scraper/url.go
View File

@ -8,23 +8,22 @@ import (
)
type Url struct {
TweetID TweetID
TweetID TweetID `db:"tweet_id"`
Domain string `db:"domain"`
Text string `db:"text"`
ShortText string `db:"short_text"`
Title string `db:"title"`
Description string `db:"description"`
ThumbnailWidth int `db:"thumbnail_width"`
ThumbnailHeight int `db:"thumbnail_height"`
ThumbnailRemoteUrl string `db:"thumbnail_remote_url"`
ThumbnailLocalPath string `db:"thumbnail_local_path"`
CreatorID UserID `db:"creator_id"`
SiteID UserID `db:"site_id"`
Domain string
Text string
ShortText string
Title string
Description string
ThumbnailWidth int
ThumbnailHeight int
ThumbnailRemoteUrl string
ThumbnailLocalPath string
CreatorID UserID
SiteID UserID
HasCard bool
HasThumbnail bool
IsContentDownloaded bool
HasCard bool `db:"has_card"`
HasThumbnail bool `db:"has_thumbnail"`
IsContentDownloaded bool `db:"is_content_downloaded"`
}
func ParseAPIUrlCard(apiCard APICard) Url {

42
scraper/user.go
View File

@ -23,31 +23,31 @@ func JoinArrayOfHandles(handles []UserHandle) string {
}
type User struct {
ID UserID
DisplayName string
Handle UserHandle
Bio string
FollowingCount int
FollowersCount int
Location string
Website string
JoinDate Timestamp
IsPrivate bool
IsVerified bool
IsBanned bool
IsDeleted bool
ProfileImageUrl string
ProfileImageLocalPath string
BannerImageUrl string
BannerImageLocalPath string
ID UserID `db:"id"`
DisplayName string `db:"display_name"`
Handle UserHandle `db:"handle"`
Bio string `db:"bio"`
FollowingCount int `db:"following_count"`
FollowersCount int `db:"followers_count"`
Location string `db:"location"`
Website string `db:"website"`
JoinDate Timestamp `db:"join_date"`
IsPrivate bool `db:"is_private"`
IsVerified bool `db:"is_verified"`
IsBanned bool `db:"is_banned"`
IsDeleted bool `db:"is_deleted"`
ProfileImageUrl string `db:"profile_image_url"`
ProfileImageLocalPath string `db:"profile_image_local_path"`
BannerImageUrl string `db:"banner_image_url"`
BannerImageLocalPath string `db:"banner_image_local_path"`
PinnedTweetID TweetID
PinnedTweetID TweetID `db:"pinned_tweet_id"`
PinnedTweet *Tweet
IsFollowed bool
IsContentDownloaded bool
IsFollowed bool `db:"is_followed"`
IsContentDownloaded bool `db:"is_content_downloaded"`
IsNeedingFakeID bool
IsIdFake bool
IsIdFake bool `db:"is_id_fake"`
}
func (u User) String() string {

24
scraper/video.go
View File

@ -12,21 +12,21 @@ type VideoID int64
// from someone else).
type Video struct {
ID VideoID
TweetID TweetID
Width int
Height int
RemoteURL string
LocalFilename string
ID VideoID `db:"id"`
TweetID TweetID `db:"tweet_id"`
Width int `db:"width"`
Height int `db:"height"`
RemoteURL string `db:"remote_url"`
LocalFilename string `db:"local_filename"`
ThumbnailRemoteUrl string
ThumbnailRemoteUrl string `db:"thumbnail_remote_url"`
ThumbnailLocalPath string `db:"thumbnail_local_filename"`
Duration int // milliseconds
ViewCount int
Duration int `db:"duration"` // milliseconds
ViewCount int `db:"view_count"`
IsDownloaded bool
IsBlockedByDMCA bool
IsGif bool
IsDownloaded bool `db:"is_downloaded"`
IsBlockedByDMCA bool `db:"is_blocked_by_dmca"`
IsGif bool `db:"is_gif"`
}
func get_filename(remote_url string) string {