From 0de9b25dfeb0237f1f57007259001fd2ca6ce332 Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 22 Nov 2021 14:56:57 -0800 Subject: [PATCH] Change 'in_reply_to' and 'quoted_tweet' ID fields to say '_id' (hungarian notation) --- persistence/schema.sql | 4 ++-- persistence/tweet_queries.go | 8 ++++---- scraper/api_types_test.go | 6 +++--- scraper/tweet.go | 26 +++++++++++++------------- scraper/tweet_test.go | 12 ++++++------ 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/persistence/schema.sql b/persistence/schema.sql index 916b17f..5d84fc7 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -40,8 +40,8 @@ create table tweets (rowid integer primary key, num_retweets integer, num_replies integer, num_quote_tweets integer, - in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id` - quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id` + in_reply_to_id integer, + quoted_tweet_id integer, mentions text, -- comma-separated reply_mentions text, -- comma-separated hashtags text, -- comma-separated diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go index 8645eb3..85436e1 100644 --- a/persistence/tweet_queries.go +++ b/persistence/tweet_queries.go @@ -16,7 +16,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { return err } _, err = db.Exec(` - insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded) + insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?) on conflict do update set num_likes=?, @@ -26,7 +26,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { is_stub=(is_stub and ?), is_content_downloaded=(is_content_downloaded or ?) `, - t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, + t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, ) @@ -84,7 +84,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { db := p.DB stmt, err := db.Prepare(` - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded + select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid where id = ? `) @@ -101,7 +101,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { var hashtags string row := stmt.QueryRow(id) - err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded) + err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded) if err != nil { return t, err } diff --git a/scraper/api_types_test.go b/scraper/api_types_test.go index d252a12..38d5c68 100644 --- a/scraper/api_types_test.go +++ b/scraper/api_types_test.go @@ -14,7 +14,7 @@ func TestNormalizeContent(t *testing.T) { filename string eventual_full_text string quoted_status_id scraper.TweetID - in_reply_to scraper.TweetID + in_reply_to_id scraper.TweetID retweeted_status_id scraper.TweetID reply_mentions string } { @@ -49,8 +49,8 @@ func TestNormalizeContent(t *testing.T) { if scraper.TweetID(tweet.QuotedStatusID) != v.quoted_status_id { t.Errorf("Expected quoted status %d, but got %d", v.quoted_status_id, tweet.QuotedStatusID) } - if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to { - t.Errorf("Expected in_reply_to id %d, but got %d", v.in_reply_to, tweet.InReplyToStatusID) + if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to_id { + t.Errorf("Expected in_reply_to_id id %d, but got %d", v.in_reply_to_id, tweet.InReplyToStatusID) } if scraper.TweetID(tweet.RetweetedStatusID) != v.retweeted_status_id { t.Errorf("Expected retweeted status id %d, but got %d", v.retweeted_status_id, tweet.RetweetedStatusID) diff --git a/scraper/tweet.go b/scraper/tweet.go index 16b27dd..ceaa29f 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -13,16 +13,17 @@ const DEFAULT_MAX_REPLIES_EAGER_LOAD = 50 type TweetID int64 type Tweet struct { - ID TweetID - UserID UserID - User *User - Text string - PostedAt time.Time - NumLikes int - NumRetweets int - NumReplies int - NumQuoteTweets int - InReplyTo TweetID + ID TweetID + UserID UserID + User *User + Text string + PostedAt time.Time + NumLikes int + NumRetweets int + NumReplies int + NumQuoteTweets int + InReplyToID TweetID + QuotedTweetID TweetID Urls []Url Images []Image @@ -30,7 +31,6 @@ type Tweet struct { Mentions []UserHandle ReplyMentions []UserHandle Hashtags []string - QuotedTweet TweetID TombstoneType string IsStub bool @@ -95,7 +95,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { ret.NumRetweets = apiTweet.RetweetCount ret.NumReplies = apiTweet.ReplyCount ret.NumQuoteTweets = apiTweet.QuoteCount - ret.InReplyTo = TweetID(apiTweet.InReplyToStatusID) + ret.InReplyToID = TweetID(apiTweet.InReplyToStatusID) for _, url := range apiTweet.Entities.URLs { var url_object Url @@ -132,7 +132,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { } } - ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID) + ret.QuotedTweetID = TweetID(apiTweet.QuotedStatusID) for _, entity := range apiTweet.ExtendedEntities.Media { if entity.Type != "video" && entity.Type != "animated_gif" { diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 63c7f90..680aeea 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -48,8 +48,8 @@ func TestParseSingleTweet(t *testing.T) { t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix()) } - if tweet.QuotedTweet != 0 { - t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweet) + if tweet.QuotedTweetID != 0 { + t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweetID) } } @@ -74,16 +74,16 @@ func TestParseTweetWithQuotedTweetAsLink(t *testing.T) { } expected_replied_id := scraper.TweetID(1395882872729477131) - if tweet.InReplyTo != expected_replied_id { - t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyTo) + if tweet.InReplyToID != expected_replied_id { + t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyToID) } if len(tweet.ReplyMentions) != 0 { t.Errorf("Wanted %v, got %v", []string{}, tweet.ReplyMentions) } expected_quoted_id := scraper.TweetID(1396194494710788100) - if tweet.QuotedTweet != expected_quoted_id { - t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweet) + if tweet.QuotedTweetID != expected_quoted_id { + t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweetID) } }