From 15241f4f43f76a94845cda56686353c6376df107 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 25 Jul 2021 14:37:32 -0700 Subject: [PATCH] Add tweet queries --- persistence/tweet_queries.go | 179 ++++++++++++++++++++++++++++++ persistence/tweet_queries_test.go | 113 +++++++++++++++++++ persistence/user_queries_test.go | 8 +- 3 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 persistence/tweet_queries.go create mode 100644 persistence/tweet_queries_test.go diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go new file mode 100644 index 0000000..7049b02 --- /dev/null +++ b/persistence/tweet_queries.go @@ -0,0 +1,179 @@ +package persistence + +import ( + "fmt" + "time" + "strings" + "database/sql" + + "offline_twitter/scraper" +) + +func (p Profile) SaveTweet(t scraper.Tweet) error { + db := p.DB + + tx, err := db.Begin() + if err != nil { + return err + } + _, err = db.Exec(` + insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, video_url, in_reply_to, quoted_tweet, mentions, hashtags) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict do update + set num_likes=?, + num_retweets=?, + num_replies=?, + num_quote_tweets=? + `, + t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.Video, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), strings.Join(t.Hashtags, ","), + t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, + ) + + if err != nil { + return err + } + for _, url := range t.Urls { + _, err := db.Exec("insert into urls (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, url) + if err != nil { + return err + } + } + for _, image := range t.Images { + _, err := db.Exec("insert into images (tweet_id, filename) values (?, ?) on conflict do nothing", t.ID, image) + if err != nil { + return err + } + } + for _, hashtag := range t.Hashtags { + _, err := db.Exec("insert into hashtags (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, hashtag) + if err != nil { + return err + } + } + err = tx.Commit() + if err != nil { + return err + } + return nil +} + +func (p Profile) IsTweetInDatabase(id scraper.TweetID) bool { + db := p.DB + + var dummy string + err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy) + if err != nil { + if err != sql.ErrNoRows { + // A real error + panic(err) + } + return false + } + return true +} + +func (p Profile) attach_images(t *scraper.Tweet) error { + println("Attaching images") + stmt, err := p.DB.Prepare("select filename from images where tweet_id = ?") + if err != nil { + return err + } + defer stmt.Close() + rows, err := stmt.Query(t.ID) + if err != nil { + return err + } + var img string + for rows.Next() { + err = rows.Scan(&img) + if err != nil { + return err + } + println(img) + t.Images = append(t.Images, img) + fmt.Printf("%v\n", t.Images) + } + return nil +} + +func (p Profile) attach_urls(t *scraper.Tweet) error { + println("Attaching urls") + stmt, err := p.DB.Prepare("select text from urls where tweet_id = ?") + if err != nil { + return err + } + defer stmt.Close() + rows, err := stmt.Query(t.ID) + if err != nil { + return err + } + var url string + for rows.Next() { + err = rows.Scan(&url) + if err != nil { + return err + } + println(url) + t.Urls = append(t.Urls, url) + fmt.Printf("%v\n", t.Urls) + } + return nil +} + +func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { + db := p.DB + + stmt, err := db.Prepare(` + select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, video_url, in_reply_to, quoted_tweet, mentions, hashtags + from tweets + where id = ? + `) + + if err != nil { + return scraper.Tweet{}, err + } + defer stmt.Close() + + var t scraper.Tweet + var postedAt int + var mentions string + var hashtags string + var tweet_id int64 + var user_id int64 + + row := stmt.QueryRow(id) + err = row.Scan(&tweet_id, &user_id, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.Video, &t.InReplyTo, &t.QuotedTweet, &mentions, &hashtags) + if err != nil { + return t, err + } + + t.PostedAt = time.Unix(int64(postedAt), 0) // args are `seconds` and `nanoseconds` + for _, m := range strings.Split(mentions, ",") { + t.Mentions = append(t.Mentions, scraper.UserHandle(m)) + } + t.Hashtags = strings.Split(hashtags, ",") + t.ID = scraper.TweetID(fmt.Sprint(tweet_id)) + t.UserID = scraper.UserID(fmt.Sprint(user_id)) + + err = p.attach_images(&t) + if err != nil { + return t, err + } + err = p.attach_urls(&t) + return t, err +} + + +func (p Profile) LoadUserFor(t *scraper.Tweet) error { + if t.User != nil { + // Already there, no need to load it + return nil + } + + user, err := p.GetUserByID(t.UserID) + if err != nil { + return err + } + t.User = &user + return nil +} diff --git a/persistence/tweet_queries_test.go b/persistence/tweet_queries_test.go new file mode 100644 index 0000000..5034f36 --- /dev/null +++ b/persistence/tweet_queries_test.go @@ -0,0 +1,113 @@ +package persistence_test + +import ( + "testing" + + "github.com/go-test/deep" + +) + +/** + * Create a Tweet, save it, reload it, and make sure it comes back the same + */ +func TestSaveAndLoadTweet(t *testing.T) { + profile_path := "test_profiles/TestTweetQueries" + profile := create_or_load_profile(profile_path) + + tweet := create_dummy_tweet() + user := create_dummy_user() + + tweet.UserID = user.ID + + // Save the user + err := profile.SaveUser(user) + if err != nil { + t.Fatalf("Failed to save the user, so no point in continuing the test: %s", err.Error()) + } + + // Save the tweet + err = profile.SaveTweet(tweet) + if err != nil { + t.Errorf("Failed to save the tweet: %s", err.Error()) + } + + // Reload the tweet + new_tweet, err := profile.GetTweetById(tweet.ID) + if err != nil { + t.Errorf("Failed to load the tweet: %s", err.Error()) + } + + if diff := deep.Equal(tweet, new_tweet); diff != nil { + t.Error(diff) + } +} + +/** + * Should correctly report whether the User exists in the database + */ +func TestIsTweetInDatabase(t *testing.T) { + profile_path := "test_profiles/TestTweetQueries" + profile := create_or_load_profile(profile_path) + + tweet := create_dummy_tweet() + user := create_dummy_user() + tweet.UserID = user.ID + + // Save the user + err := profile.SaveUser(user) + if err != nil { + t.Fatalf("Failed to save the user, so no point in continuing the test: %s", err.Error()) + } + + exists := profile.IsTweetInDatabase(tweet.ID) + if exists { + t.Errorf("It shouldn't exist, but it does: %s", tweet.ID) + } + err = profile.SaveTweet(tweet) + if err != nil { + panic(err) + } + exists = profile.IsTweetInDatabase(tweet.ID) + if !exists { + t.Errorf("It should exist, but it doesn't: %s", tweet.ID) + } +} + +/** + * Should correctly populate the `User` field on a Tweet + */ +func TestLoadUserForTweet(t *testing.T) { + profile_path := "test_profiles/TestTweetQueries" + profile := create_or_load_profile(profile_path) + + tweet := create_dummy_tweet() + user := create_dummy_user() + + tweet.UserID = user.ID + + // Save the user + err := profile.SaveUser(user) + if err != nil { + t.Fatalf("Failed to save the user, so no point in continuing the test: %s", err.Error()) + } + + // Save the tweet + err = profile.SaveTweet(tweet) + if err != nil { + t.Errorf("Failed to save the tweet: %s", err.Error()) + } + + + if tweet.User != nil { + t.Errorf("`User` field is already there for some reason: %v", tweet.User) + } + + err = profile.LoadUserFor(&tweet) + if err != nil { + t.Errorf("Failed to load user: %s", err.Error()) + } + + if tweet.User == nil { + t.Errorf("Did not load a user. It is still nil.") + } +} diff --git a/persistence/user_queries_test.go b/persistence/user_queries_test.go index 3778b2d..706e548 100644 --- a/persistence/user_queries_test.go +++ b/persistence/user_queries_test.go @@ -4,8 +4,6 @@ import ( "testing" "github.com/go-test/deep" - - "offline_twitter/scraper" ) @@ -33,7 +31,7 @@ func TestSaveAndLoadUser(t *testing.T) { } // Same thing, but get by handle - new_fake_user2, err := profile.GetUserByHandle(scraper.UserHandle(fake_user.Handle)) + new_fake_user2, err := profile.GetUserByHandle(fake_user.Handle) if err != nil { panic(err) } @@ -53,7 +51,7 @@ func TestUserExists(t *testing.T) { user := create_dummy_user() - exists := profile.UserExists(scraper.UserHandle(user.Handle)) + exists := profile.UserExists(user.Handle) if exists { t.Errorf("It shouldn't exist, but it does: %s", user.ID) } @@ -61,7 +59,7 @@ func TestUserExists(t *testing.T) { if err != nil { panic(err) } - exists = profile.UserExists(scraper.UserHandle(user.Handle)) + exists = profile.UserExists(user.Handle) if !exists { t.Errorf("It should exist, but it doesn't: %s", user.ID) }