me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: create SaveTweetTrove convenience method to reduce code in main.go

Browse Source
This commit is contained in:
Alessio 2022-02-14 13:39:59 -08:00
parent 3d489a8e95
commit f9a8841c07
3 changed files with 61 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/twitter/helpers.go
View File

@ -78,6 +78,14 @@ func die(text string, display_help bool, exit_code int) {
os.Exit(exit_code)
}
/**
* Print a happy exit message and exit
*/
func happy_exit(text string) {
fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n")
fmt.Printf(terminal_utils.COLOR_GREEN + "Exiting successfully." + terminal_utils.COLOR_RESET + "\n")
}
/**
* Helper function - parse a tweet permalink URL to extract the tweet ID
*

95
cmd/twitter/main.go
View File

@ -127,9 +127,8 @@ func fetch_user(handle scraper.UserHandle) {
die("Error saving user: " + err.Error(), false, 4)
}
fmt.Println("Saved the user. Downloading content..")
download_user_content(handle);
happy_exit("Saved the user")
}
/**
@ -157,7 +156,7 @@ func fetch_tweet_only(tweet_identifier string) {
if err != nil {
die("Error saving tweet: " + err.Error(), false, 4)
}
fmt.Println("Saved the tweet. Exiting successfully")
happy_exit("Saved the tweet")
}
/**
@ -180,32 +179,9 @@ func fetch_tweet_conversation(tweet_identifier string) {
if err != nil {
die(err.Error(), false, -1)
}
tweets, _, users := trove.Transform()
profile.SaveTweetTrove(trove)
for _, u := range users {
fmt.Println(u.Handle)
err = profile.DownloadUserProfileImageTiny(&u)
if err != nil {
die("Error getting user content: " + err.Error(), false, 10)
}
err = profile.SaveUser(u)
if err != nil {
die("Error saving user: " + err.Error(), false, 4)
}
}
for _, t := range tweets {
err = profile.SaveTweet(t)
if err != nil {
die(fmt.Sprintf("Error saving tweet (id %d): %s", t.ID, err.Error()), false, 4)
}
err = profile.DownloadTweetContentFor(&t)
if err != nil {
die("Error getting tweet content: " + err.Error(), false, 11)
}
}
fmt.Printf("Saved %d tweets and %d users. Exiting successfully\n", len(tweets), len(users))
happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users)))
}
/**
@ -220,44 +196,13 @@ func fetch_user_feed(handle string, how_many int) {
die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1)
}
// tweets, retweets, users, err := scraper.GetUserFeedFor(user.ID, how_many);
trove, err := scraper.GetUserFeedGraphqlFor(user.ID, how_many)
if err != nil {
die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2)
}
tweets, retweets, users := trove.Transform();
profile.SaveTweetTrove(trove)
for _, u := range users {
fmt.Println(u.Handle)
err = profile.DownloadUserProfileImageTiny(&u)
if err != nil {
die("Error getting user content: " + err.Error(), false, 10)
}
err = profile.SaveUser(u)
if err != nil {
die("Error saving user: " + err.Error(), false, 4)
}
}
for _, t := range tweets {
err = profile.SaveTweet(t)
if err != nil {
die("Error saving tweet: " + err.Error(), false, 4)
}
err = profile.DownloadTweetContentFor(&t)
if err != nil {
die("Error getting tweet content: " + err.Error(), false, 11)
}
}
for _, r := range retweets {
err = profile.SaveRetweet(r)
if err != nil {
die("Error saving retweet: " + err.Error(), false, 4)
}
}
fmt.Printf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(tweets), len(retweets), len(users))
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
}
@ -294,31 +239,7 @@ func search(query string) {
if err != nil {
die("Error scraping search results: " + err.Error(), false, -100)
}
tweets, retweets, users := trove.Transform()
profile.SaveTweetTrove(trove)
for _, u := range users {
fmt.Println(u.Handle)
err = profile.DownloadUserProfileImageTiny(&u)
if err != nil {
die("Error getting user content: " + err.Error(), false, 10)
}
err = profile.SaveUser(u)
if err != nil {
die("Error saving user: " + err.Error(), false, 4)
}
}
for _, t := range tweets {
err = profile.SaveTweet(t)
if err != nil {
die("Error saving tweet: " + err.Error(), false, 4)
}
err = profile.DownloadTweetContentFor(&t)
if err != nil {
die("Error getting tweet content: " + err.Error(), false, 11)
}
}
fmt.Printf("Saved %d tweets, %d retweets and %d users. Exiting successfully\n", len(tweets), len(retweets), len(users))
happy_exit(fmt.Sprintf("Saved %d tweets and %d users. Exiting successfully\n", len(trove.Tweets), len(trove.Users)))
}

45
persistence/tweet_trove_queries.go Normal file
View File

@ -0,0 +1,45 @@
package persistence
import (
"fmt"
. "offline_twitter/scraper"
)
/**
* Convenience function that saves all the objects in a TweetTrove.
* Panics if anything goes wrong.
*/
func (p Profile) SaveTweetTrove(trove TweetTrove) {
for _, u := range trove.Users {
// Download download their tiny profile image
err := p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Sprintf("Error downloading user content for user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error()))
}
err = p.SaveUser(u)
if err != nil {
panic(fmt.Sprintf("Error saving user with ID %d and handle %s: %s", u.ID, u.Handle, err.Error()))
}
}
for _, t := range trove.Tweets {
err := p.SaveTweet(t)
if err != nil {
panic(fmt.Sprintf("Error saving tweet ID %d: %s", t.ID, err.Error()))
}
err = p.DownloadTweetContentFor(&t)
if err != nil {
panic(fmt.Sprintf("Error downloading tweet content for tweet ID %d: %s", t.ID, err.Error()))
}
}
for _, r := range trove.Retweets {
err := p.SaveRetweet(r)
if err != nil {
panic(fmt.Sprintf("Error saving retweet with ID %d from user ID %d: %s", r.RetweetID, r.RetweetedByID, err.Error()))
}
}
}