me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change 'in_reply_to' and 'quoted_tweet' ID fields to say '_id' (hungarian notation)

Browse Source
This commit is contained in:
Alessio 2021-11-22 14:56:57 -08:00
parent d0cb857acb
commit 0de9b25dfe
5 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence/schema.sql
View File

@ -40,8 +40,8 @@ create table tweets (rowid integer primary key,
num_retweets integer, num_retweets integer,
num_replies integer, num_replies integer,
num_quote_tweets integer, num_quote_tweets integer,
in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id` in_reply_to_id integer,
quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id` quoted_tweet_id integer,
mentions text, -- comma-separated mentions text, -- comma-separated
reply_mentions text, -- comma-separated reply_mentions text, -- comma-separated
hashtags text, -- comma-separated hashtags text, -- comma-separated

8
persistence/tweet_queries.go
View File

@ -16,7 +16,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
return err return err
} }
_, err = db.Exec(` _, err = db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded) insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?)
on conflict do update on conflict do update
set num_likes=?, set num_likes=?,
@ -26,7 +26,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
is_stub=(is_stub and ?), is_stub=(is_stub and ?),
is_content_downloaded=(is_content_downloaded or ?) is_content_downloaded=(is_content_downloaded or ?)
`, `,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded,
) )
@ -84,7 +84,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded
from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where id = ? where id = ?
`) `)
@ -101,7 +101,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
var hashtags string var hashtags string
row := stmt.QueryRow(id) row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded) err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded)
if err != nil { if err != nil {
return t, err return t, err
} }

6
scraper/api_types_test.go
View File

@ -14,7 +14,7 @@ func TestNormalizeContent(t *testing.T) {
filename string filename string
eventual_full_text string eventual_full_text string
quoted_status_id scraper.TweetID quoted_status_id scraper.TweetID
in_reply_to scraper.TweetID in_reply_to_id scraper.TweetID
retweeted_status_id scraper.TweetID retweeted_status_id scraper.TweetID
reply_mentions string reply_mentions string
} { } {
@ -49,8 +49,8 @@ func TestNormalizeContent(t *testing.T) {
if scraper.TweetID(tweet.QuotedStatusID) != v.quoted_status_id { if scraper.TweetID(tweet.QuotedStatusID) != v.quoted_status_id {
t.Errorf("Expected quoted status %d, but got %d", v.quoted_status_id, tweet.QuotedStatusID) t.Errorf("Expected quoted status %d, but got %d", v.quoted_status_id, tweet.QuotedStatusID)
} }
if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to { if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to_id {
t.Errorf("Expected in_reply_to id %d, but got %d", v.in_reply_to, tweet.InReplyToStatusID) t.Errorf("Expected in_reply_to_id id %d, but got %d", v.in_reply_to_id, tweet.InReplyToStatusID)
} }
if scraper.TweetID(tweet.RetweetedStatusID) != v.retweeted_status_id { if scraper.TweetID(tweet.RetweetedStatusID) != v.retweeted_status_id {
t.Errorf("Expected retweeted status id %d, but got %d", v.retweeted_status_id, tweet.RetweetedStatusID) t.Errorf("Expected retweeted status id %d, but got %d", v.retweeted_status_id, tweet.RetweetedStatusID)

8
scraper/tweet.go
View File

@ -22,7 +22,8 @@ type Tweet struct {
NumRetweets int NumRetweets int
NumReplies int NumReplies int
NumQuoteTweets int NumQuoteTweets int
InReplyTo TweetID InReplyToID TweetID
QuotedTweetID TweetID
Urls []Url Urls []Url
Images []Image Images []Image
@ -30,7 +31,6 @@ type Tweet struct {
Mentions []UserHandle Mentions []UserHandle
ReplyMentions []UserHandle ReplyMentions []UserHandle
Hashtags []string Hashtags []string
QuotedTweet TweetID
TombstoneType string TombstoneType string
IsStub bool IsStub bool
@ -95,7 +95,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
ret.NumRetweets = apiTweet.RetweetCount ret.NumRetweets = apiTweet.RetweetCount
ret.NumReplies = apiTweet.ReplyCount ret.NumReplies = apiTweet.ReplyCount
ret.NumQuoteTweets = apiTweet.QuoteCount ret.NumQuoteTweets = apiTweet.QuoteCount
ret.InReplyTo = TweetID(apiTweet.InReplyToStatusID) ret.InReplyToID = TweetID(apiTweet.InReplyToStatusID)
for _, url := range apiTweet.Entities.URLs { for _, url := range apiTweet.Entities.URLs {
var url_object Url var url_object Url
@ -132,7 +132,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
} }
} }
ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID) ret.QuotedTweetID = TweetID(apiTweet.QuotedStatusID)
for _, entity := range apiTweet.ExtendedEntities.Media { for _, entity := range apiTweet.ExtendedEntities.Media {
if entity.Type != "video" && entity.Type != "animated_gif" { if entity.Type != "video" && entity.Type != "animated_gif" {

12
scraper/tweet_test.go
View File

@ -48,8 +48,8 @@ func TestParseSingleTweet(t *testing.T) {
t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix()) t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix())
} }
if tweet.QuotedTweet != 0 { if tweet.QuotedTweetID != 0 {
t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweet) t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweetID)
} }
} }
@ -74,16 +74,16 @@ func TestParseTweetWithQuotedTweetAsLink(t *testing.T) {
} }
expected_replied_id := scraper.TweetID(1395882872729477131) expected_replied_id := scraper.TweetID(1395882872729477131)
if tweet.InReplyTo != expected_replied_id { if tweet.InReplyToID != expected_replied_id {
t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyTo) t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyToID)
} }
if len(tweet.ReplyMentions) != 0 { if len(tweet.ReplyMentions) != 0 {
t.Errorf("Wanted %v, got %v", []string{}, tweet.ReplyMentions) t.Errorf("Wanted %v, got %v", []string{}, tweet.ReplyMentions)
} }
expected_quoted_id := scraper.TweetID(1396194494710788100) expected_quoted_id := scraper.TweetID(1396194494710788100)
if tweet.QuotedTweet != expected_quoted_id { if tweet.QuotedTweetID != expected_quoted_id {
t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweet) t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweetID)
} }
} }