Add proper logging module, set logging level to 'info' by default (hide very verbose debug results)

cmd/twitter/main.go

@@ -4,6 +4,9 @@ import (
 	"os"
 	"fmt"
 	"flag"
+
+	log "github.com/sirupsen/logrus"
+
 	"offline_twitter/scraper"
 	"offline_twitter/persistence"
 )
@@ -28,6 +31,14 @@ func main() {
 	how_many := flag.Int("n", 50, "")
 	flag.IntVar(how_many, "number", 50, "")
 
+	var default_log_level string
+	if version_string == "" {
+		default_log_level = "debug"
+	} else {
+		default_log_level = "info"
+	}
+	log_level := flag.String("log-level", default_log_level, "")
+
 	help := flag.Bool("help", false, "")
 	flag.BoolVar(help, "h", false, "")
 
@@ -51,6 +62,12 @@ func main() {
 		die("", true, 0)
 	}
 
+	logging_level, err := log.ParseLevel(*log_level)
+	if err != nil {
+		die(err.Error(), false, 1)
+	}
+	log.SetLevel(logging_level)
+
 	if len(args) < 2 {
 		die("", true, 1)
 	}
@@ -63,7 +80,6 @@ func main() {
 		return
 	}
 
-	var err error
 	profile, err = persistence.LoadProfile(*profile_dir)
 	if err != nil {
 		die("Could not load profile: " + err.Error(), true, 2)
@@ -113,14 +129,11 @@ func create_profile(target_dir string) {
  * - handle: e.g., "michaelmalice"
  */
 func fetch_user(handle scraper.UserHandle) {
-	if profile.UserExists(handle) {
-		fmt.Println("User is already in database.  Updating user...")
-	}
 	user, err := scraper.GetUser(handle)
 	if err != nil {
 		die(err.Error(), false, -1)
 	}
-	fmt.Println(user)
+	log.Debug(user)
 
 	err = profile.SaveUser(user)
 	if err != nil {
@@ -143,14 +156,11 @@ func fetch_tweet_only(tweet_identifier string) {
 		die(err.Error(), false, -1)
 	}
 
-	if profile.IsTweetInDatabase(tweet_id) {
-		fmt.Println("Tweet is already in database.  Updating...")
-	}
 	tweet, err := scraper.GetTweet(tweet_id)
 	if err != nil {
 		die("Error fetching tweet: " + err.Error(), false, -1)
 	}
-	fmt.Println(tweet)
+	log.Debug(tweet)
 
 	err = profile.SaveTweet(tweet)
 	if err != nil {
@@ -171,17 +181,13 @@ func fetch_tweet_conversation(tweet_identifier string) {
 		die(err.Error(), false, -1)
 	}
 
-	if profile.IsTweetInDatabase(tweet_id) {
-		fmt.Println("Tweet is already in database.  Updating...")
-	}
-
 	trove, err := scraper.GetTweetFull(tweet_id)
 	if err != nil {
 		die(err.Error(), false, -1)
 	}
 	profile.SaveTweetTrove(trove)
 
-	happy_exit(fmt.Sprintf("Saved %d tweets and %d users.  Exiting successfully\n", len(trove.Tweets), len(trove.Users)))
+	happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
 }
 
 /**
@@ -202,7 +208,7 @@ func fetch_user_feed(handle string, how_many int) {
 	}
 	profile.SaveTweetTrove(trove)
 
-	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users.  Exiting successfully\n", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
+	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
 }
 
 
@@ -241,5 +247,5 @@ func search(query string) {
 	}
 	profile.SaveTweetTrove(trove)
 
-	happy_exit(fmt.Sprintf("Saved %d tweets and %d users.  Exiting successfully\n", len(trove.Tweets), len(trove.Users)))
+	happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
 }

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/go-test/deep v1.0.7
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/mattn/go-sqlite3 v1.14.7
+	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -1,5 +1,6 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
 github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
@@ -8,9 +9,14 @@ github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEg
 github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

scraper/api_request_utils.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 	"net/url"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const API_CONVERSATION_BASE_PATH = "https://twitter.com/i/api/2/timeline/conversation/"
@@ -47,13 +49,18 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error)
 
 	if resp.StatusCode != http.StatusOK {
 		content, _ := ioutil.ReadAll(resp.Body)
-		return TweetResponse{}, fmt.Errorf("HTTP %s: %s", resp.Status, content)
+		s := ""
+		for header := range resp.Header {
+			s += fmt.Sprintf("    %s: %s\n", header, resp.Header.Get(header))
+		}
+		return TweetResponse{}, fmt.Errorf("HTTP %s\n%s\n%s", resp.Status, s, content)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return TweetResponse{}, err
 	}
+	log.Debug(string(body))
 
 	var response TweetResponse
 	err = json.Unmarshal(body, &response)
@@ -134,6 +141,7 @@ func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
 	if err != nil {
 		return TweetResponse{}, err
 	}
+	log.Debug(string(body))
 
 	var response TweetResponse
 	err = json.Unmarshal(body, &response)
@@ -202,6 +210,7 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) {
 		if err != nil {
 			return APIUser{}, err
 		}
+		log.Debug(string(body))
 
 		err = json.Unmarshal(body, &response)
 		if err != nil {

scraper/api_types_v2.go

@@ -8,6 +8,8 @@ import (
 	"time"
 	"encoding/json"
 	"strings"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type CardValue struct {
@@ -392,7 +394,7 @@ func (api API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response,
 	if err != nil {
 		return APIV2Response{}, err
 	}
-	fmt.Println(string(body))
+	log.Debug(string(body))
 
 	var response APIV2Response
 	err = json.Unmarshal(body, &response)

scraper/tweet_trove.go

@@ -2,6 +2,8 @@ package scraper
 
 import (
 	"fmt"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type TweetTrove struct {
@@ -77,6 +79,7 @@ func (trove *TweetTrove) FetchTombstoneUsers() {
 			continue
 		}
 
+		log.Debug("Getting tombstone user: " + handle)
 		user, err := GetUser(handle)
 		if err != nil {
 			panic(fmt.Sprintf("Error getting tombstoned user: %s\n  %s", handle, err.Error()))