diff --git a/cmd/tests.sh b/cmd/tests.sh index cb0aa8c..69aed60 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -81,6 +81,8 @@ test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_i test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429584239570391042") = "michaelmalice" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429583672827465730") = "kanesays23" test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429616911315345414") = "NovaValentis" +test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429585423702052867") = "michaelmalice" +test $(sqlite3 twitter.db "select reply_mentions from tweets where id = 1429616911315345414") = "RememberAfghan1,michaelmalice" # Test that the `--profile` flag works diff --git a/persistence/schema.sql b/persistence/schema.sql index 362a89f..d4b1976 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -34,8 +34,9 @@ create table tweets (rowid integer primary key, num_quote_tweets integer, in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id` quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id` - mentions text, -- comma-separated - hashtags text, -- comma-separated + mentions text, -- comma-separated + reply_mentions text, -- comma-separated + hashtags text, -- comma-separated is_content_downloaded boolean default 0, foreign key(user_id) references users(id) diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go index a050fb7..de474a1 100644 --- a/persistence/tweet_queries.go +++ b/persistence/tweet_queries.go @@ -16,8 +16,8 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { return err } _, err = db.Exec(` - insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set num_likes=?, num_retweets=?, @@ -25,7 +25,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { num_quote_tweets=?, is_content_downloaded=? `, - t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded, + t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsContentDownloaded, ) @@ -83,7 +83,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { db := p.DB stmt, err := db.Prepare(` - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, hashtags, is_content_downloaded + select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, is_content_downloaded from tweets where id = ? `) @@ -96,10 +96,11 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { var t scraper.Tweet var postedAt int var mentions string + var reply_mentions string var hashtags string row := stmt.QueryRow(id) - err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &hashtags, &t.IsContentDownloaded) + err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.IsContentDownloaded) if err != nil { return t, err } @@ -108,6 +109,9 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { for _, m := range strings.Split(mentions, ",") { t.Mentions = append(t.Mentions, scraper.UserHandle(m)) } + for _, m := range strings.Split(reply_mentions, ",") { + t.ReplyMentions = append(t.ReplyMentions, scraper.UserHandle(m)) + } t.Hashtags = strings.Split(hashtags, ",") imgs, err := p.GetImagesForTweet(t) diff --git a/persistence/utils_test.go b/persistence/utils_test.go index 55b9133..070f0de 100644 --- a/persistence/utils_test.go +++ b/persistence/utils_test.go @@ -213,6 +213,7 @@ func create_dummy_tweet() scraper.Tweet { Urls: []scraper.Url{url1, url2}, Images: []scraper.Image{img1, img2}, Mentions: []scraper.UserHandle{"mention1", "mention2"}, + ReplyMentions: []scraper.UserHandle{"replymention1", "replymention2"}, Hashtags: []string{"hash1", "hash2"}, } }