diff --git a/.golangci.yaml b/.golangci.yaml index 4c21c8d..38ebfff 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -13,6 +13,7 @@ linters: disable-all: true enable: - deadcode + - depguard - errcheck - gosimple - govet @@ -342,14 +343,14 @@ linters-settings: - shadow -# depguard: -# list-type: blacklist -# include-go-root: false -# packages: -# - github.com/sirupsen/logrus -# packages-with-error-message: -# # specify an error message to output when a blacklisted package is used -# - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + depguard: + list-type: blacklist + include-go-root: true + packages: + - io/ioutil + packages-with-error-message: + # specify an error message to output when a blacklisted package is used + - io/ioutil: "replace with the matching functions from `io` or `os` packages" # ifshort: # # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax. diff --git a/persistence/media_download.go b/persistence/media_download.go index d525a46..1d733e5 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -2,7 +2,7 @@ package persistence import ( "fmt" - "io/ioutil" + "io" "net/http" "os" "path" @@ -34,7 +34,7 @@ func (d DefaultDownloader) Curl(url string, outpath string) error { return fmt.Errorf("Error %s: %s", url, resp.Status) } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("Error downloading image %s:\n %w", url, err) } diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index 7399d55..2399881 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -3,7 +3,7 @@ package scraper import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "time" @@ -41,7 +41,7 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } @@ -52,7 +52,7 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) return TweetResponse{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return TweetResponse{}, fmt.Errorf("Error reading response body for GetUserFeedFor(%d):\n %w", user_id, err) } @@ -131,14 +131,14 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) { defer resp.Body.Close() if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return TweetResponse{}, fmt.Errorf("Error getting %q. HTTP %s: %s", req.URL, resp.Status, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return TweetResponse{}, fmt.Errorf("Error reading HTTP request:\n %w", err) } @@ -209,14 +209,14 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) { // Sometimes it randomly gives 403 Forbidden. API's fault, not ours // We check for this below if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return APIUser{}, fmt.Errorf("response status %s: %s", resp.Status, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return APIUser{}, fmt.Errorf("Error retrieving API response to GetUser(%s):\n %w", handle, err) } @@ -270,14 +270,14 @@ func (api API) Search(query string, cursor string) (TweetResponse, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return TweetResponse{}, fmt.Errorf("Error while searching for %q. HTTP %s: %s", req.URL, resp.Status, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return TweetResponse{}, fmt.Errorf("Error retrieving API response for Search(%q):\n %w", query, err) } diff --git a/scraper/api_types_test.go b/scraper/api_types_test.go index 8415205..10e8bf6 100644 --- a/scraper/api_types_test.go +++ b/scraper/api_types_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "github.com/stretchr/testify/assert" @@ -42,7 +42,7 @@ func TestNormalizeContent(t *testing.T) { } for _, v := range test_cases { - data, err := ioutil.ReadFile(v.filename) + data, err := os.ReadFile(v.filename) if err != nil { panic(err) } @@ -63,7 +63,7 @@ func TestNormalizeContent(t *testing.T) { func TestUserProfileToAPIUser(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/michael_malice_user_profile.json") + data, err := os.ReadFile("test_responses/michael_malice_user_profile.json") if err != nil { panic(err) } @@ -79,7 +79,7 @@ func TestUserProfileToAPIUser(t *testing.T) { func TestGetCursor(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/midriffs_anarchist_cookbook.json") + data, err := os.ReadFile("test_responses/midriffs_anarchist_cookbook.json") if err != nil { panic(err) } @@ -102,7 +102,7 @@ func TestIsEndOfFeed(t *testing.T) { {"test_responses/kwiber_end_of_feed.json", true}, } for _, v := range test_cases { - data, err := ioutil.ReadFile(v.filename) + data, err := os.ReadFile(v.filename) if err != nil { panic(err) } @@ -116,7 +116,7 @@ func TestIsEndOfFeed(t *testing.T) { func TestHandleTombstonesHidden(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tombstones/tombstone_hidden_1.json") + data, err := os.ReadFile("test_responses/tombstones/tombstone_hidden_1.json") if err != nil { panic(err) } @@ -146,7 +146,7 @@ func TestHandleTombstonesHidden(t *testing.T) { func TestHandleTombstonesDeleted(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tombstones/tombstone_deleted.json") + data, err := os.ReadFile("test_responses/tombstones/tombstone_deleted.json") if err != nil { panic(err) } @@ -169,7 +169,7 @@ func TestHandleTombstonesDeleted(t *testing.T) { func TestHandleTombstonesUnavailable(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tombstones/tombstone_unavailable.json") + data, err := os.ReadFile("test_responses/tombstones/tombstone_unavailable.json") if err != nil { panic(err) } diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 2f5e576..7ec0b9c 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -1,13 +1,13 @@ package scraper import ( + "encoding/json" "fmt" + "io" "net/http" "net/url" - "io/ioutil" - "time" - "encoding/json" "strings" + "time" log "github.com/sirupsen/logrus" ) @@ -397,7 +397,7 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } @@ -408,7 +408,7 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, return APIV2Response{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return APIV2Response{}, fmt.Errorf("Error reading HTTP response body:\n %w", err) } diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index 0f7d17e..7dc2697 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "fmt" @@ -16,7 +16,7 @@ import ( * Parse an APIV2User */ func TestAPIV2ParseUser(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/user_michael_malice.json") + data, err := os.ReadFile("test_responses/api_v2/user_michael_malice.json") if err != nil { panic(err) } @@ -53,7 +53,7 @@ func TestAPIV2ParseUser(t *testing.T) { * Parse a plain text tweet */ func TestAPIV2ParseTweet(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_plaintext.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_plaintext.json") if err != nil { panic(err) } @@ -102,7 +102,7 @@ func TestAPIV2ParseTweet(t *testing.T) { */ func TestAPIV2ParseTweetWithQuotedTweet(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_quoted_tweet.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_quoted_tweet.json") if err != nil { panic(err) } @@ -156,7 +156,7 @@ func TestAPIV2ParseTweetWithQuotedTweet(t *testing.T) { */ func TestAPIV2ParseRetweet(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/retweet.json") + data, err := os.ReadFile("test_responses/api_v2/retweet.json") if err != nil { panic(err) } @@ -216,7 +216,7 @@ func TestAPIV2ParseRetweet(t *testing.T) { */ func TestAPIV2ParseRetweetedQuoteTweet(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/retweet_with_quote_tweet.json") + data, err := os.ReadFile("test_responses/api_v2/retweet_with_quote_tweet.json") if err != nil { panic(err) } @@ -276,7 +276,7 @@ func TestAPIV2ParseRetweetedQuoteTweet(t *testing.T) { */ func TestAPIV2ParseTweetWithQuotedTombstone(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_quoted_tombstone.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_quoted_tombstone.json") if err != nil { panic(err) } @@ -312,7 +312,7 @@ func TestAPIV2ParseTweetWithQuotedTombstone(t *testing.T) { */ func TestAPIV2ParseTweetWithURL(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_url.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_url.json") if err != nil { panic(err) } @@ -349,7 +349,7 @@ func TestAPIV2ParseTweetWithURL(t *testing.T) { */ func TestAPIV2ParseTweetWithURLPlayerCard(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_url_player_card.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_url_player_card.json") if err != nil { panic(err) } @@ -381,7 +381,7 @@ func TestAPIV2ParseTweetWithURLPlayerCard(t *testing.T) { */ func TestAPIV2ParseTweetWithURLRetweet(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/retweet_with_url.json") + data, err := os.ReadFile("test_responses/api_v2/retweet_with_url.json") if err != nil { panic(err) } @@ -408,7 +408,7 @@ func TestAPIV2ParseTweetWithURLRetweet(t *testing.T) { */ func TestAPIV2ParseTweetWithPoll(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_poll.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_poll.json") if err != nil { panic(err) } @@ -444,7 +444,7 @@ func TestAPIV2ParseTweetWithPoll(t *testing.T) { func TestParseAPIV2UserFeed(t *testing.T) { - data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json") + data, err := os.ReadFile("test_responses/api_v2/user_feed_apiv2.json") if err != nil { panic(err) } @@ -501,7 +501,7 @@ func TestParseAPIV2UserFeed(t *testing.T) { */ func TestAPIV2FeedIsEmpty(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/empty_response.json") + data, err := os.ReadFile("test_responses/api_v2/empty_response.json") if err != nil { panic(err) } @@ -528,7 +528,7 @@ func TestAPIV2FeedIsEmpty(t *testing.T) { */ func TestAPIV2GetMainInstructionFromFeed(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json") + data, err := os.ReadFile("test_responses/api_v2/user_feed_apiv2.json") if err != nil { panic(err) } @@ -555,7 +555,7 @@ func TestAPIV2GetMainInstructionFromFeed(t *testing.T) { */ func TestAPIV2TombstoneEntry(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tombstone_tweet.json") + data, err := os.ReadFile("test_responses/api_v2/tombstone_tweet.json") require.NoError(t, err) var tweet_result APIV2Result @@ -571,7 +571,7 @@ func TestAPIV2TombstoneEntry(t *testing.T) { func TestTweetWithWarning(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/api_v2/tweet_with_warning.json") + data, err := os.ReadFile("test_responses/api_v2/tweet_with_warning.json") require.NoError(t, err) var tweet_result APIV2Result err = json.Unmarshal(data, &tweet_result) diff --git a/scraper/guest_token.go b/scraper/guest_token.go index 87551db..1095307 100644 --- a/scraper/guest_token.go +++ b/scraper/guest_token.go @@ -1,11 +1,12 @@ package scraper -import "fmt" -import "time" - -import "io/ioutil" -import "net/http" -import "encoding/json" +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) type GuestTokenResponse struct { Token string `json:"guest_token"` @@ -34,14 +35,14 @@ func GetGuestToken() (string, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - content, err := ioutil.ReadAll(resp.Body) + content, err := io.ReadAll(resp.Body) if err != nil { panic(err) } return "", fmt.Errorf("HTTP %s: %s", resp.Status, content) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("Error reading HTTP response body:\n %w", err) } diff --git a/scraper/guest_token_test.go b/scraper/guest_token_test.go index e506950..b8b1335 100644 --- a/scraper/guest_token_test.go +++ b/scraper/guest_token_test.go @@ -1,8 +1,9 @@ package scraper_test import ( - "testing" "fmt" + "testing" + . "offline_twitter/scraper" ) @@ -19,11 +20,10 @@ func TestGetGuestToken(t *testing.T) { fmt.Println(token) } - // Tests the caching. Should run much much faster than an HTTP request, since all requests // other than the first use the cache. func BenchmarkGetGuestToken(b *testing.B) { for i := 0; i < b.N; i++ { - GetGuestToken() // nolint:errcheck // Don't care about errors, just want to time it + GetGuestToken() // nolint:errcheck // Don't care about errors, just want to time it } } diff --git a/scraper/image_test.go b/scraper/image_test.go index cb15fee..e4ee7d4 100644 --- a/scraper/image_test.go +++ b/scraper/image_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "github.com/stretchr/testify/assert" @@ -13,7 +13,7 @@ import ( func TestParseAPIMedia(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/image.json") + data, err := os.ReadFile("test_responses/tweet_content/image.json") if err != nil { panic(err) } diff --git a/scraper/poll_test.go b/scraper/poll_test.go index c4dd34f..bf3e442 100644 --- a/scraper/poll_test.go +++ b/scraper/poll_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "github.com/stretchr/testify/assert" @@ -13,7 +13,7 @@ import ( func TestParsePoll2Choices(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/poll_card_2_options.json") + data, err := os.ReadFile("test_responses/tweet_content/poll_card_2_options.json") if err != nil { panic(err) } @@ -37,7 +37,7 @@ func TestParsePoll2Choices(t *testing.T) { func TestParsePoll4Choices(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/poll_card_4_options_ended.json") + data, err := os.ReadFile("test_responses/tweet_content/poll_card_4_options_ended.json") if err != nil { panic(err) } diff --git a/scraper/retweet_test.go b/scraper/retweet_test.go index db5317a..eb9b57e 100644 --- a/scraper/retweet_test.go +++ b/scraper/retweet_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "encoding/json" - "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" @@ -13,7 +13,7 @@ import ( func TestParseSingleRetweet(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_that_is_a_retweet.json") + data, err := os.ReadFile("test_responses/tweet_that_is_a_retweet.json") if err != nil { panic(err) } diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 2a86863..8491494 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "encoding/json" - "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" @@ -12,7 +12,7 @@ import ( ) func load_tweet_from_file(filename string) Tweet{ - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { panic(err) } @@ -174,7 +174,7 @@ func TestTweetWithPoll(t *testing.T) { func TestParseTweetResponse(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/michael_malice_feed.json") + data, err := os.ReadFile("test_responses/michael_malice_feed.json") if err != nil { panic(err) } @@ -193,7 +193,7 @@ func TestParseTweetResponse(t *testing.T) { func TestParseTweetResponseWithTombstones(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tombstones/tombstone_deleted.json") + data, err := os.ReadFile("test_responses/tombstones/tombstone_deleted.json") if err != nil { panic(err) } diff --git a/scraper/url_test.go b/scraper/url_test.go index c4166d1..e186935 100644 --- a/scraper/url_test.go +++ b/scraper/url_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "github.com/stretchr/testify/assert" @@ -13,7 +13,7 @@ import ( func TestParseAPIUrlCard(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/url_card.json") + data, err := os.ReadFile("test_responses/tweet_content/url_card.json") if err != nil { panic(err) } @@ -38,7 +38,7 @@ func TestParseAPIUrlCard(t *testing.T) { func TestParseAPIUrlCardWithPlayer(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/url_card_with_player.json") + data, err := os.ReadFile("test_responses/tweet_content/url_card_with_player.json") if err != nil { panic(err) } @@ -60,7 +60,7 @@ func TestParseAPIUrlCardWithPlayer(t *testing.T) { func TestParseAPIUrlCardWithPlayerAndPlaceholderThumbnail(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/url_card_with_player_placeholder_image.json") + data, err := os.ReadFile("test_responses/tweet_content/url_card_with_player_placeholder_image.json") if err != nil { panic(err) } @@ -82,7 +82,7 @@ func TestParseAPIUrlCardWithPlayerAndPlaceholderThumbnail(t *testing.T) { func TestParseAPIUrlCardWithoutThumbnail(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/url_card_without_thumbnail.json") + data, err := os.ReadFile("test_responses/tweet_content/url_card_without_thumbnail.json") if err != nil { panic(err) } diff --git a/scraper/user_test.go b/scraper/user_test.go index bd8a923..05e9d09 100644 --- a/scraper/user_test.go +++ b/scraper/user_test.go @@ -3,7 +3,7 @@ package scraper_test import ( "testing" "encoding/json" - "io/ioutil" + "os" "net/http" "github.com/jarcoal/httpmock" @@ -15,7 +15,7 @@ import ( func TestParseSingleUser(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/michael_malice_user_profile.json") + data, err := os.ReadFile("test_responses/michael_malice_user_profile.json") if err != nil { panic(err) } @@ -54,7 +54,7 @@ func TestParseSingleUser(t *testing.T) { */ func TestParseBannedUser(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/suspended_user.json") + data, err := os.ReadFile("test_responses/suspended_user.json") if err != nil { panic(err) } @@ -79,7 +79,7 @@ func TestParseBannedUser(t *testing.T) { */ func TestParseDeletedUser(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/deleted_user.json") + data, err := os.ReadFile("test_responses/deleted_user.json") if err != nil { panic(err) } diff --git a/scraper/video_test.go b/scraper/video_test.go index bb20919..79adce8 100644 --- a/scraper/video_test.go +++ b/scraper/video_test.go @@ -2,7 +2,7 @@ package scraper_test import ( "testing" - "io/ioutil" + "os" "encoding/json" "github.com/stretchr/testify/assert" @@ -13,7 +13,7 @@ import ( func TestParseAPIVideo(t *testing.T) { assert := assert.New(t) - data, err := ioutil.ReadFile("test_responses/tweet_content/video.json") + data, err := os.ReadFile("test_responses/tweet_content/video.json") if err != nil { panic(err) }