From f9a8841c077201709bc02e4bae878f7c3fff6f4c Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 14 Feb 2022 13:39:59 -0800 Subject: [PATCH] REFACTOR: create `SaveTweetTrove` convenience method to reduce code in `main.go` --- cmd/twitter/helpers.go | 8 +++ cmd/twitter/main.go | 95 +++--------------------------- persistence/tweet_trove_queries.go | 45 ++++++++++++++ 3 files changed, 61 insertions(+), 87 deletions(-) create mode 100644 persistence/tweet_trove_queries.go diff --git a/cmd/twitter/helpers.go b/cmd/twitter/helpers.go index 11614c9..7b5749c 100644 --- a/cmd/twitter/helpers.go +++ b/cmd/twitter/helpers.go @@ -78,6 +78,14 @@ func die(text string, display_help bool, exit_code int) { os.Exit(exit_code) } +/** + * Print a happy exit message and exit + */ +func happy_exit(text string) { + fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n") + fmt.Printf(terminal_utils.COLOR_GREEN + "Exiting successfully." + terminal_utils.COLOR_RESET + "\n") +} + /** * Helper function - parse a tweet permalink URL to extract the tweet ID * diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 62d59b4..8db017d 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -127,9 +127,8 @@ func fetch_user(handle scraper.UserHandle) { die("Error saving user: " + err.Error(), false, 4) } - fmt.Println("Saved the user. Downloading content..") - download_user_content(handle); + happy_exit("Saved the user") } /** @@ -157,7 +156,7 @@ func fetch_tweet_only(tweet_identifier string) { if err != nil { die("Error saving tweet: " + err.Error(), false, 4) } - fmt.Println("Saved the tweet. Exiting successfully") + happy_exit("Saved the tweet") } /** @@ -180,32 +179,9 @@ func fetch_tweet_conversation(tweet_identifier string) { if err != nil { die(err.Error(), false, -1) } - tweets, _, users := trove.Transform() + profile.SaveTweetTrove(trove) - for _, u := range users { - fmt.Println(u.Handle) - err = profile.DownloadUserProfileImageTiny(&u) - if err != nil { - die("Error getting user content: " + err.Error(), false, 10) - } - - err = profile.SaveUser(u) - if err != nil { - die("Error saving user: " + err.Error(), false, 4) - } - } - - for _, t := range tweets { - err = profile.SaveTweet(t) - if err != nil { - die(fmt.Sprintf("Error saving tweet (id %d): %s", t.ID, err.Error()), false, 4) - } - err = profile.DownloadTweetContentFor(&t) - if err != nil { - die("Error getting tweet content: " + err.Error(), false, 11) - } - } - fmt.Printf("Saved %d tweets and %d users. Exiting successfully\n", len(tweets), len(users)) + happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users))) } /** @@ -220,44 +196,13 @@ func fetch_user_feed(handle string, how_many int) { die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) } - // tweets, retweets, users, err := scraper.GetUserFeedFor(user.ID, how_many); trove, err := scraper.GetUserFeedGraphqlFor(user.ID, how_many) if err != nil { die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) } - tweets, retweets, users := trove.Transform(); + profile.SaveTweetTrove(trove) - for _, u := range users { - fmt.Println(u.Handle) - err = profile.DownloadUserProfileImageTiny(&u) - if err != nil { - die("Error getting user content: " + err.Error(), false, 10) - } - err = profile.SaveUser(u) - if err != nil { - die("Error saving user: " + err.Error(), false, 4) - } - } - - for _, t := range tweets { - err = profile.SaveTweet(t) - if err != nil { - die("Error saving tweet: " + err.Error(), false, 4) - } - err = profile.DownloadTweetContentFor(&t) - if err != nil { - die("Error getting tweet content: " + err.Error(), false, 11) - } - } - - for _, r := range retweets { - err = profile.SaveRetweet(r) - if err != nil { - die("Error saving retweet: " + err.Error(), false, 4) - } - } - - fmt.Printf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(tweets), len(retweets), len(users)) + happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } @@ -294,31 +239,7 @@ func search(query string) { if err != nil { die("Error scraping search results: " + err.Error(), false, -100) } - tweets, retweets, users := trove.Transform() + profile.SaveTweetTrove(trove) - for _, u := range users { - fmt.Println(u.Handle) - err = profile.DownloadUserProfileImageTiny(&u) - if err != nil { - die("Error getting user content: " + err.Error(), false, 10) - } - - err = profile.SaveUser(u) - if err != nil { - die("Error saving user: " + err.Error(), false, 4) - } - } - - for _, t := range tweets { - err = profile.SaveTweet(t) - if err != nil { - die("Error saving tweet: " + err.Error(), false, 4) - } - err = profile.DownloadTweetContentFor(&t) - if err != nil { - die("Error getting tweet content: " + err.Error(), false, 11) - } - } - - fmt.Printf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(tweets), len(retweets), len(users)) + happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users))) } diff --git a/persistence/tweet_trove_queries.go b/persistence/tweet_trove_queries.go new file mode 100644 index 0000000..a770e10 --- /dev/null +++ b/persistence/tweet_trove_queries.go @@ -0,0 +1,45 @@ +package persistence + +import ( + "fmt" + + . "offline_twitter/scraper" +) + +/** + * Convenience function that saves all the objects in a TweetTrove. + * Panics if anything goes wrong. + */ +func (p Profile) SaveTweetTrove(trove TweetTrove) { + for _, u := range trove.Users { + // Download download their tiny profile image + err := p.DownloadUserProfileImageTiny(&u) + if err != nil { + panic(fmt.Sprintf("Error downloading user content for user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error())) + } + + err = p.SaveUser(u) + if err != nil { + panic(fmt.Sprintf("Error saving user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error())) + } + } + + for _, t := range trove.Tweets { + err := p.SaveTweet(t) + if err != nil { + panic(fmt.Sprintf("Error saving tweet ID %d: %s", t.ID, err.Error())) + } + + err = p.DownloadTweetContentFor(&t) + if err != nil { + panic(fmt.Sprintf("Error downloading tweet content for tweet ID %d: %s", t.ID, err.Error())) + } + } + + for _, r := range trove.Retweets { + err := p.SaveRetweet(r) + if err != nil { + panic(fmt.Sprintf("Error saving retweet with ID %d from user ID %d: %s", r.RetweetID, r.RetweetedByID, err.Error())) + } + } +}