Enable 'wrapcheck' linter

This commit is contained in:
Alessio 2022-03-13 16:13:16 -07:00
parent 8261087103
commit f08da27cc2
13 changed files with 167 additions and 136 deletions

View File

@ -23,7 +23,7 @@ linters:
- unused - unused
- varcheck - varcheck
- whitespace - whitespace
# - wrapcheck - wrapcheck
- lll - lll
- godox - godox
- errorlint - errorlint
@ -533,17 +533,11 @@ linters-settings:
# multi-if: false # Enforces newlines (or comments) after every multi-line if statement # multi-if: false # Enforces newlines (or comments) after every multi-line if statement
# multi-func: false # Enforces newlines (or comments) after every multi-line function signature # multi-func: false # Enforces newlines (or comments) after every multi-line function signature
# wrapcheck: wrapcheck:
# # An array of strings that specify substrings of signatures to ignore. # An array of strings that specify substrings of signatures to ignore.
# # If this set, it will override the default set of ignored signatures. # If this set, it will override the default set of ignored signatures.
# # See https://github.com/tomarrell/wrapcheck#configuration for more information. # See https://github.com/tomarrell/wrapcheck#configuration for more information.
# ignoreSigs: ignoreSigs:
# - .Errorf(
# - errors.New(
# - errors.Unwrap(
# - .Wrap(
# - .Wrapf(
# - .WithMessage(
# # The custom section can be used to define linter plugins to be loaded at runtime. # # The custom section can be used to define linter plugins to be loaded at runtime.
# # See README doc for more info. # # See README doc for more info.

View File

@ -28,7 +28,7 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
println(url) println(url)
resp, err := http.Get(url) resp, err := http.Get(url)
if err != nil { if err != nil {
return err return fmt.Errorf("Error executing HTTP GET(%q):\n %w", url, err)
} }
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return fmt.Errorf("Error %s: %s", url, resp.Status) return fmt.Errorf("Error %s: %s", url, resp.Status)
@ -53,7 +53,7 @@ func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownlo
outfile := path.Join(p.ProfileDir, "images", img.LocalFilename) outfile := path.Join(p.ProfileDir, "images", img.LocalFilename)
err := downloader.Curl(img.RemoteURL, outfile) err := downloader.Curl(img.RemoteURL, outfile)
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading tweet image (TweetID %d):\n %w", img.TweetID, err)
} }
img.IsDownloaded = true img.IsDownloaded = true
return p.SaveImage(*img) return p.SaveImage(*img)
@ -67,14 +67,14 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename) outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
err := downloader.Curl(v.RemoteURL, outfile) err := downloader.Curl(v.RemoteURL, outfile)
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading video (TweetID %d):\n %w", v.TweetID, err)
} }
// Download the thumbnail // Download the thumbnail
outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath) outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath)
err = downloader.Curl(v.ThumbnailRemoteUrl, outfile) err = downloader.Curl(v.ThumbnailRemoteUrl, outfile)
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading video thumbnail (TweetID %d):\n %w", v.TweetID, err)
} }
v.IsDownloaded = true v.IsDownloaded = true
@ -89,7 +89,7 @@ func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownl
outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath)
err := downloader.Curl(url.ThumbnailRemoteUrl, outfile) err := downloader.Curl(url.ThumbnailRemoteUrl, outfile)
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading link thumbnail (TweetID %d):\n %w", url.TweetID, err)
} }
} }
url.IsContentDownloaded = true url.IsContentDownloaded = true
@ -166,7 +166,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med
err := downloader.Curl(target_url, outfile) err := downloader.Curl(target_url, outfile)
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading profile image for user %q:\n %w", u.Handle, err)
} }
// Skip it if there's no banner image // Skip it if there's no banner image
@ -179,7 +179,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med
err = downloader.Curl(u.BannerImageUrl+"/600x200", outfile) err = downloader.Curl(u.BannerImageUrl+"/600x200", outfile)
} }
if err != nil { if err != nil {
return err return fmt.Errorf("Error downloading banner image for user %q:\n %w", u.Handle, err)
} }
} }

View File

@ -1,6 +1,8 @@
package persistence package persistence
import ( import (
"fmt"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -20,7 +22,10 @@ func (p Profile) SaveImage(img scraper.Image) error {
img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded, img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
img.IsDownloaded, img.IsDownloaded,
) )
return err if err != nil {
return fmt.Errorf("Error saving image (tweet ID %d):\n %w", img.TweetID, err)
}
return nil
} }
/** /**
@ -43,7 +48,10 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
vid.IsDownloaded, vid.ViewCount, vid.IsDownloaded, vid.ViewCount,
) )
return err if err != nil {
return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err)
}
return nil
} }
/** /**
@ -62,7 +70,10 @@ func (p Profile) SaveUrl(url scraper.Url) error {
url.IsContentDownloaded, url.IsContentDownloaded,
) )
return err if err != nil {
return fmt.Errorf("Error saving Url (tweet ID %d):\n %w", url.TweetID, err)
}
return nil
} }
/** /**
@ -85,7 +96,10 @@ func (p Profile) SavePoll(poll scraper.Poll) error {
poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt, poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt,
) )
return err if err != nil {
return fmt.Errorf("Error saving Poll (tweet ID %d):\n %w", poll.TweetID, err)
}
return nil
} }
/** /**

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"os" "os"
"path" "path"
sql "github.com/jmoiron/sqlx" sql "github.com/jmoiron/sqlx"
"github.com/jmoiron/sqlx/reflectx" "github.com/jmoiron/sqlx/reflectx"
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
@ -51,7 +52,7 @@ func NewProfile(target_dir string) (Profile, error) {
fmt.Printf("Creating new profile: %s\n", target_dir) fmt.Printf("Creating new profile: %s\n", target_dir)
err := os.Mkdir(target_dir, os.FileMode(0755)) err := os.Mkdir(target_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating directory %q:\n %w", target_dir, err)
} }
// Create `twitter.db` // Create `twitter.db`
@ -66,46 +67,46 @@ func NewProfile(target_dir string) (Profile, error) {
settings := Settings{} settings := Settings{}
data, err := yaml.Marshal(&settings) data, err := yaml.Marshal(&settings)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error YAML-marshalling [empty!] settings file:\n %w", err)
} }
err = os.WriteFile(settings_file, data, os.FileMode(0644)) err = os.WriteFile(settings_file, data, os.FileMode(0644))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating settings file %q:\n %w", settings_file, err)
} }
// Create `profile_images` // Create `profile_images`
fmt.Printf("Creating............. %s/\n", profile_images_dir) fmt.Printf("Creating............. %s/\n", profile_images_dir)
err = os.Mkdir(profile_images_dir, os.FileMode(0755)) err = os.Mkdir(profile_images_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating %q:\n %w", profile_images_dir, err)
} }
// Create `link_thumbnail_images` // Create `link_thumbnail_images`
fmt.Printf("Creating............. %s/\n", link_thumbnails_dir) fmt.Printf("Creating............. %s/\n", link_thumbnails_dir)
err = os.Mkdir(link_thumbnails_dir, os.FileMode(0755)) err = os.Mkdir(link_thumbnails_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating %q:\n %w", link_thumbnails_dir, err)
} }
// Create `images` // Create `images`
fmt.Printf("Creating............. %s/\n", images_dir) fmt.Printf("Creating............. %s/\n", images_dir)
err = os.Mkdir(images_dir, os.FileMode(0755)) err = os.Mkdir(images_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating %q:\n %w", images_dir, err)
} }
// Create `videos` // Create `videos`
fmt.Printf("Creating............. %s/\n", videos_dir) fmt.Printf("Creating............. %s/\n", videos_dir)
err = os.Mkdir(videos_dir, os.FileMode(0755)) err = os.Mkdir(videos_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating %q:\n %w", videos_dir, err)
} }
// Create `video_thumbnails` // Create `video_thumbnails`
fmt.Printf("Creating............. %s/\n", video_thumbnails_dir) fmt.Printf("Creating............. %s/\n", video_thumbnails_dir)
err = os.Mkdir(video_thumbnails_dir, os.FileMode(0755)) err = os.Mkdir(video_thumbnails_dir, os.FileMode(0755))
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error creating %q:\n %w", video_thumbnails_dir, err)
} }
return Profile{target_dir, settings, db}, nil return Profile{target_dir, settings, db}, nil
@ -135,12 +136,12 @@ func LoadProfile(profile_dir string) (Profile, error) {
settings_data, err := os.ReadFile(settings_file) settings_data, err := os.ReadFile(settings_file)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error reading %q:\n %w", settings_file, err)
} }
settings := Settings{} settings := Settings{}
err = yaml.Unmarshal(settings_data, &settings) err = yaml.Unmarshal(settings_data, &settings)
if err != nil { if err != nil {
return Profile{}, err return Profile{}, fmt.Errorf("Error YAML-unmarshalling %q:\n %w", settings_file, err)
} }
db := sql.MustOpen("sqlite3", fmt.Sprintf("%s?_foreign_keys=on&_journal_mode=WAL", sqlite_file)) db := sql.MustOpen("sqlite3", fmt.Sprintf("%s?_foreign_keys=on&_journal_mode=WAL", sqlite_file))

View File

@ -1,6 +1,8 @@
package persistence package persistence
import ( import (
"fmt"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -15,7 +17,10 @@ func (p Profile) SaveRetweet(r scraper.Retweet) error {
`, `,
r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(), r.RetweetID, r.TweetID, r.RetweetedByID, r.RetweetedAt.Unix(),
) )
return err if err != nil {
return fmt.Errorf("Error executing SaveRetweet(%d):\n %w", r.RetweetID, err)
}
return nil
} }
/** /**
@ -28,5 +33,8 @@ func (p Profile) GetRetweetById(id scraper.TweetID) (scraper.Retweet, error) {
from retweets from retweets
where retweet_id = ? where retweet_id = ?
`, id) `, id)
return r, err if err != nil {
return r, fmt.Errorf("Error executing GetRetweetById(%d):\n %w", id, err)
}
return r, nil
} }

View File

@ -2,8 +2,9 @@ package persistence
import ( import (
"database/sql" "database/sql"
"strings"
"errors" "errors"
"fmt"
"strings"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -37,7 +38,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
) )
if err != nil { if err != nil {
return err return fmt.Errorf("Error executing SaveTweet(ID %d):\n %w", t.ID, err)
} }
for _, url := range t.Urls { for _, url := range t.Urls {
err := p.SaveUrl(url) err := p.SaveUrl(url)
@ -60,7 +61,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
for _, hashtag := range t.Hashtags { for _, hashtag := range t.Hashtags {
_, err := db.Exec("insert into hashtags (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, hashtag) _, err := db.Exec("insert into hashtags (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, hashtag)
if err != nil { if err != nil {
return err return fmt.Errorf("Error inserting hashtag %q on tweet ID %d:\n %w", hashtag, t.ID, err)
} }
} }
for _, poll := range t.Polls { for _, poll := range t.Polls {
@ -72,7 +73,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
err = tx.Commit() err = tx.Commit()
if err != nil { if err != nil {
return err return fmt.Errorf("Error committing SaveTweet transaction:\n %w", err)
} }
return nil return nil
} }
@ -104,7 +105,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
`) `)
if err != nil { if err != nil {
return scraper.Tweet{}, err return scraper.Tweet{}, fmt.Errorf("Error preparing statement in GetTweetByID(%d):\n %w", id, err)
} }
defer stmt.Close() defer stmt.Close()
@ -118,7 +119,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
&t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded,
&t.IsConversationScraped, &t.LastScrapedAt) &t.IsConversationScraped, &t.LastScrapedAt)
if err != nil { if err != nil {
return t, err return t, fmt.Errorf("Error parsing result in GetTweetByID(%d):\n %w", id, err)
} }
t.Mentions = []scraper.UserHandle{} t.Mentions = []scraper.UserHandle{}

View File

@ -1,9 +1,10 @@
package persistence package persistence
import ( import (
"fmt"
"errors"
"database/sql" "database/sql"
"errors"
"fmt"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -59,7 +60,7 @@ func (p Profile) SaveUser(u *scraper.User) error {
u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded,
) )
if err != nil { if err != nil {
return err return fmt.Errorf("Error executing SaveUser(%s):\n %w", u.Handle, err)
} }
return nil return nil
@ -140,7 +141,10 @@ func (p Profile) GetUserByID(id scraper.UserID) (scraper.User, error) {
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return ret, ErrNotInDatabase{"User", id} return ret, ErrNotInDatabase{"User", id}
} }
return ret, err if err != nil {
panic(err)
}
return ret, nil
} }
/** /**

View File

@ -87,7 +87,7 @@ func (p Profile) GetDatabaseVersion() (int, error) {
err := row.Scan(&version) err := row.Scan(&version)
if err != nil { if err != nil {
return 0, err return 0, fmt.Errorf("Error checking database version:\n %w", err)
} }
return version, nil return version, nil
} }

View File

@ -20,12 +20,12 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error)
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_USER_TIMELINE_BASE_PATH, user_id), nil) req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_USER_TIMELINE_BASE_PATH, user_id), nil)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error initializing HTTP request for GetFeedFor(%d):\n %w", user_id, err)
} }
err = ApiRequestAddTokens(req) err = ApiRequestAddTokens(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err)
} }
ApiRequestAddAllParams(req) ApiRequestAddAllParams(req)
@ -36,7 +36,7 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error)
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error executing HTTP request for GetFeedFor(%d):\n %w", user_id, err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -54,13 +54,16 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error)
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error reading response body for GetUserFeedFor(%d):\n %w", user_id, err)
} }
log.Debug(string(body)) log.Debug(string(body))
var response TweetResponse var response TweetResponse
err = json.Unmarshal(body, &response) err = json.Unmarshal(body, &response)
return response, err if err != nil {
return response, fmt.Errorf("Error parsing API response for GetUserFeedFor(%d):\n %w", user_id, err)
}
return response, nil
} }
/** /**
@ -104,17 +107,16 @@ func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, mi
return nil return nil
} }
func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) { func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id), nil) req, err := http.NewRequest("GET", fmt.Sprintf("%s%d.json", API_CONVERSATION_BASE_PATH, id), nil)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error initializing HTTP request:\n %w", err)
} }
err = ApiRequestAddTokens(req) err = ApiRequestAddTokens(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err)
} }
ApiRequestAddAllParams(req) ApiRequestAddAllParams(req)
@ -124,7 +126,7 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error executing HTTP request:\n %w", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -138,13 +140,16 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error reading HTTP request:\n %w", err)
} }
log.Debug(string(body)) log.Debug(string(body))
var response TweetResponse var response TweetResponse
err = json.Unmarshal(body, &response) err = json.Unmarshal(body, &response)
return response, err if err != nil {
return response, fmt.Errorf("Error parsing API response for GetTweet(%d):\n %w", id, err)
}
return response, nil
} }
// Resend the request to get more replies if necessary // Resend the request to get more replies if necessary
@ -178,27 +183,26 @@ func UpdateQueryCursor(req *http.Request, new_cursor string, is_tweet bool) {
req.URL.RawQuery = query.Encode() req.URL.RawQuery = query.Encode()
} }
func (api API) GetUser(handle UserHandle) (APIUser, error) { func (api API) GetUser(handle UserHandle) (APIUser, error) {
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest( req, err := http.NewRequest(
"GET", "GET",
"https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22" + string(handle) + "https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22"+string(handle)+
"%22%2C%22withHighlightedLabel%22%3Atrue%7D", "%22%2C%22withHighlightedLabel%22%3Atrue%7D",
nil) nil)
if err != nil { if err != nil {
return APIUser{}, err return APIUser{}, fmt.Errorf("Error initializing HTTP request:\n %w", err)
} }
err = ApiRequestAddTokens(req) err = ApiRequestAddTokens(req)
if err != nil { if err != nil {
return APIUser{}, err return APIUser{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err)
} }
var response UserResponse var response UserResponse
for retries := 0; retries < 3; retries += 1 { for retries := 0; retries < 3; retries += 1 {
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return APIUser{}, err return APIUser{}, fmt.Errorf("Error executing HTTP request for GetUser(%s):\n %w", handle, err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -214,13 +218,13 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return APIUser{}, err return APIUser{}, fmt.Errorf("Error retrieving API response to GetUser(%s):\n %w", handle, err)
} }
log.Debug(string(body)) log.Debug(string(body))
err = json.Unmarshal(body, &response) err = json.Unmarshal(body, &response)
if err != nil { if err != nil {
return APIUser{}, err return APIUser{}, fmt.Errorf("Error parsing API response to GetUser(%s):\n %w", handle, err)
} }
// Retry ONLY if the error is code 50 (random authentication failure), NOT on real errors // Retry ONLY if the error is code 50 (random authentication failure), NOT on real errors
@ -240,16 +244,16 @@ func (api API) Search(query string, cursor string) (TweetResponse, error) {
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest( req, err := http.NewRequest(
"GET", "GET",
"https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q=" + "https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q="+
url.QueryEscape(query), url.QueryEscape(query),
nil) nil)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error initializing HTTP request:\n %w", err)
} }
err = ApiRequestAddTokens(req) err = ApiRequestAddTokens(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err)
} }
ApiRequestAddAllParams(req) ApiRequestAddAllParams(req)
@ -261,7 +265,7 @@ func (api API) Search(query string, cursor string) (TweetResponse, error) {
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error executing HTTP request for Search(%q):\n %w", query, err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -275,13 +279,16 @@ func (api API) Search(query string, cursor string) (TweetResponse, error) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, fmt.Errorf("Error retrieving API response for Search(%q):\n %w", query, err)
} }
// fmt.Println(string(body)) // fmt.Println(string(body))
var response TweetResponse var response TweetResponse
err = json.Unmarshal(body, &response) err = json.Unmarshal(body, &response)
return response, err if err != nil {
return response, fmt.Errorf("Error parsing API response to Search(%q):\n %w", query, err)
}
return response, nil
} }
func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, max_results int) error { func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, max_results int) error {
@ -311,10 +318,9 @@ func (api API) GetMoreTweetsFromSearch(query string, response *TweetResponse, ma
return nil return nil
} }
// Add Bearer token and guest token // Add Bearer token and guest token
func ApiRequestAddTokens(req *http.Request) error { func ApiRequestAddTokens(req *http.Request) error {
req.Header.Set("Authorization", "Bearer " + BEARER_TOKEN) req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
req.Header.Set("x-twitter-client-language", "en") req.Header.Set("x-twitter-client-language", "en")
guestToken, err := GetGuestToken() guestToken, err := GetGuestToken()

View File

@ -378,12 +378,12 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response,
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", get_graphql_user_timeline_url(user_id, cursor), nil) req, err := http.NewRequest("GET", get_graphql_user_timeline_url(user_id, cursor), nil)
if err != nil { if err != nil {
return APIV2Response{}, err return APIV2Response{}, fmt.Errorf("Error initializing HTTP request:\n %w", err)
} }
err = ApiRequestAddTokens(req) err = ApiRequestAddTokens(req)
if err != nil { if err != nil {
return APIV2Response{}, err return APIV2Response{}, fmt.Errorf("Error adding tokens to HTTP request:\n %w", err)
} }
if cursor != "" { if cursor != "" {
@ -392,7 +392,7 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response,
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return APIV2Response{}, err return APIV2Response{}, fmt.Errorf("Error executing HTTP request:\n %w", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -410,13 +410,16 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response,
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return APIV2Response{}, err return APIV2Response{}, fmt.Errorf("Error reading HTTP response body:\n %w", err)
} }
log.Debug(string(body)) log.Debug(string(body))
var response APIV2Response var response APIV2Response
err = json.Unmarshal(body, &response) err = json.Unmarshal(body, &response)
return response, err if err != nil {
return response, fmt.Errorf("Error parsing API response for GetGraphqlFeedFor(%d):\n %w", user_id, err)
}
return response, nil
} }
/** /**

View File

@ -22,13 +22,13 @@ func GetGuestToken() (string, error) {
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("Error initializing HTTP request:\n %w", err)
} }
req.Header.Set("Authorization", "Bearer " + BEARER_TOKEN) req.Header.Set("Authorization", "Bearer "+BEARER_TOKEN)
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("Error executing HTTP request:\n %w", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -43,12 +43,12 @@ func GetGuestToken() (string, error) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("Error reading HTTP response body:\n %w", err)
} }
err = json.Unmarshal(body, &guestToken) err = json.Unmarshal(body, &guestToken)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("Error parsing API response:\n %w", err)
} }
guestToken.RefreshedAt = time.Now() guestToken.RefreshedAt = time.Now()

View File

@ -1,9 +1,9 @@
package scraper package scraper
import ( import (
"time"
"fmt"
"database/sql/driver" "database/sql/driver"
"fmt"
"time"
) )
type Timestamp struct { type Timestamp struct {
@ -32,7 +32,7 @@ func TimestampFromString(s string) (Timestamp, error) {
if err == nil { if err == nil {
return Timestamp{tmp}, nil return Timestamp{tmp}, nil
} }
return Timestamp{}, err return Timestamp{}, fmt.Errorf("Error parsing timestamp:\n %w", err)
} }
func TimestampFromUnix(num int64) Timestamp { func TimestampFromUnix(num int64) Timestamp {

View File

@ -200,7 +200,7 @@ func GetTweet(id TweetID) (Tweet, error) {
single_tweet, ok := tweet_response.GlobalObjects.Tweets[fmt.Sprint(id)] single_tweet, ok := tweet_response.GlobalObjects.Tweets[fmt.Sprint(id)]
if !ok { if !ok {
return Tweet{}, fmt.Errorf("Didn't get the tweet!\n%v", tweet_response) return Tweet{}, fmt.Errorf("Didn't get the tweet!")
} }
return ParseSingleTweet(single_tweet) return ParseSingleTweet(single_tweet)