From 9cd7f4171a14fec424b246c89f0e31c97ec8f215 Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 14 Feb 2022 13:54:18 -0800 Subject: [PATCH] Add proper logging module, set logging level to 'info' by default (hide very verbose debug results) --- cmd/twitter/main.go | 38 +++++++++++++++++++++--------------- go.mod | 1 + go.sum | 8 +++++++- scraper/api_request_utils.go | 11 ++++++++++- scraper/api_types_v2.go | 4 +++- scraper/tweet_trove.go | 3 +++ 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 8db017d..b15ddc6 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -4,6 +4,9 @@ import ( "os" "fmt" "flag" + + log "github.com/sirupsen/logrus" + "offline_twitter/scraper" "offline_twitter/persistence" ) @@ -28,6 +31,14 @@ func main() { how_many := flag.Int("n", 50, "") flag.IntVar(how_many, "number", 50, "") + var default_log_level string + if version_string == "" { + default_log_level = "debug" + } else { + default_log_level = "info" + } + log_level := flag.String("log-level", default_log_level, "") + help := flag.Bool("help", false, "") flag.BoolVar(help, "h", false, "") @@ -51,6 +62,12 @@ func main() { die("", true, 0) } + logging_level, err := log.ParseLevel(*log_level) + if err != nil { + die(err.Error(), false, 1) + } + log.SetLevel(logging_level) + if len(args) < 2 { die("", true, 1) } @@ -63,7 +80,6 @@ func main() { return } - var err error profile, err = persistence.LoadProfile(*profile_dir) if err != nil { die("Could not load profile: " + err.Error(), true, 2) @@ -113,14 +129,11 @@ func create_profile(target_dir string) { * - handle: e.g., "michaelmalice" */ func fetch_user(handle scraper.UserHandle) { - if profile.UserExists(handle) { - fmt.Println("User is already in database. Updating user...") - } user, err := scraper.GetUser(handle) if err != nil { die(err.Error(), false, -1) } - fmt.Println(user) + log.Debug(user) err = profile.SaveUser(user) if err != nil { @@ -143,14 +156,11 @@ func fetch_tweet_only(tweet_identifier string) { die(err.Error(), false, -1) } - if profile.IsTweetInDatabase(tweet_id) { - fmt.Println("Tweet is already in database. Updating...") - } tweet, err := scraper.GetTweet(tweet_id) if err != nil { die("Error fetching tweet: " + err.Error(), false, -1) } - fmt.Println(tweet) + log.Debug(tweet) err = profile.SaveTweet(tweet) if err != nil { @@ -171,17 +181,13 @@ func fetch_tweet_conversation(tweet_identifier string) { die(err.Error(), false, -1) } - if profile.IsTweetInDatabase(tweet_id) { - fmt.Println("Tweet is already in database. Updating...") - } - trove, err := scraper.GetTweetFull(tweet_id) if err != nil { die(err.Error(), false, -1) } profile.SaveTweetTrove(trove) - happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users))) + happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users))) } /** @@ -202,7 +208,7 @@ func fetch_user_feed(handle string, how_many int) { } profile.SaveTweetTrove(trove) - happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) + happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } @@ -241,5 +247,5 @@ func search(query string) { } profile.SaveTweetTrove(trove) - happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users))) + happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users))) } diff --git a/go.mod b/go.mod index 606d592..bdcd5ad 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-test/deep v1.0.7 github.com/jarcoal/httpmock v1.1.0 github.com/mattn/go-sqlite3 v1.14.7 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index c871256..df38f49 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE= @@ -8,9 +9,14 @@ github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEg github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index 1680f2f..9d47d12 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -7,6 +7,8 @@ import ( "net/http" "net/url" "time" + + log "github.com/sirupsen/logrus" ) const API_CONVERSATION_BASE_PATH = "https://twitter.com/i/api/2/timeline/conversation/" @@ -47,13 +49,18 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error) if resp.StatusCode != http.StatusOK { content, _ := ioutil.ReadAll(resp.Body) - return TweetResponse{}, fmt.Errorf("HTTP %s: %s", resp.Status, content) + s := "" + for header := range resp.Header { + s += fmt.Sprintf(" %s: %s\n", header, resp.Header.Get(header)) + } + return TweetResponse{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return TweetResponse{}, err } + log.Debug(string(body)) var response TweetResponse err = json.Unmarshal(body, &response) @@ -134,6 +141,7 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) { if err != nil { return TweetResponse{}, err } + log.Debug(string(body)) var response TweetResponse err = json.Unmarshal(body, &response) @@ -202,6 +210,7 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) { if err != nil { return APIUser{}, err } + log.Debug(string(body)) err = json.Unmarshal(body, &response) if err != nil { diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index 83266fd..8bb0c6d 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -8,6 +8,8 @@ import ( "time" "encoding/json" "strings" + + log "github.com/sirupsen/logrus" ) type CardValue struct { @@ -392,7 +394,7 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, if err != nil { return APIV2Response{}, err } - fmt.Println(string(body)) + log.Debug(string(body)) var response APIV2Response err = json.Unmarshal(body, &response) diff --git a/scraper/tweet_trove.go b/scraper/tweet_trove.go index 5abc04e..387ae5e 100644 --- a/scraper/tweet_trove.go +++ b/scraper/tweet_trove.go @@ -2,6 +2,8 @@ package scraper import ( "fmt" + + log "github.com/sirupsen/logrus" ) type TweetTrove struct { @@ -77,6 +79,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() { continue } + log.Debug("Getting tombstone user: " + handle) user, err := GetUser(handle) if err != nil { panic(fmt.Sprintf("Error getting tombstoned user: %s\n %s", handle, err.Error()))