me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change 'in_reply_to' and 'quoted_tweet' ID fields to say '_id' (hungarian notation)

Browse Source
This commit is contained in:
Alessio 2021-11-22 14:56:57 -08:00
parent d0cb857acb
commit 0de9b25dfe
5 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence/schema.sql
View File

@ -40,8 +40,8 @@ create table tweets (rowid integer primary key,
num_retweets integer, num_retweets integer,
num_replies integer, num_replies integer,
num_quote_tweets integer, num_quote_tweets integer,
in_reply_to integer, -- TODO hungarian: should be `in_reply_to_id` in_reply_to_id integer,
quoted_tweet integer, -- TODO hungarian: should be `quoted_tweet_id` quoted_tweet_id integer,
mentions text, -- comma-separated mentions text, -- comma-separated
reply_mentions text, -- comma-separated reply_mentions text, -- comma-separated
hashtags text, -- comma-separated hashtags text, -- comma-separated

8
persistence/tweet_queries.go
View File

@ -16,7 +16,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
return err return err
} }
_, err = db.Exec(` _, err = db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded) insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?)
on conflict do update on conflict do update
set num_likes=?, set num_likes=?,
@ -26,7 +26,7 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
is_stub=(is_stub and ?), is_stub=(is_stub and ?),
is_content_downloaded=(is_content_downloaded or ?) is_content_downloaded=(is_content_downloaded or ?)
`, `,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyTo, t.QuotedTweet, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded,
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded,
) )
@ -84,7 +84,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to, quoted_tweet, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded
from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where id = ? where id = ?
`) `)
@ -101,7 +101,7 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
var hashtags string var hashtags string
row := stmt.QueryRow(id) row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyTo, &t.QuotedTweet, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded) err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded)
if err != nil { if err != nil {
return t, err return t, err
} }

6
scraper/api_types_test.go
View File

@ -14,7 +14,7 @@ func TestNormalizeContent(t *testing.T) {
filename string filename string
eventual_full_text string eventual_full_text string
quoted_status_id scraper.TweetID quoted_status_id scraper.TweetID
in_reply_to scraper.TweetID in_reply_to_id scraper.TweetID
retweeted_status_id scraper.TweetID retweeted_status_id scraper.TweetID
reply_mentions string reply_mentions string
} { } {
@ -49,8 +49,8 @@ func TestNormalizeContent(t *testing.T) {
if scraper.TweetID(tweet.QuotedStatusID) != v.quoted_status_id { if scraper.TweetID(tweet.QuotedStatusID) != v.quoted_status_id {
t.Errorf("Expected quoted status %d, but got %d", v.quoted_status_id, tweet.QuotedStatusID) t.Errorf("Expected quoted status %d, but got %d", v.quoted_status_id, tweet.QuotedStatusID)
} }
if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to { if scraper.TweetID(tweet.InReplyToStatusID) != v.in_reply_to_id {
t.Errorf("Expected in_reply_to id %d, but got %d", v.in_reply_to, tweet.InReplyToStatusID) t.Errorf("Expected in_reply_to_id id %d, but got %d", v.in_reply_to_id, tweet.InReplyToStatusID)
} }
if scraper.TweetID(tweet.RetweetedStatusID) != v.retweeted_status_id { if scraper.TweetID(tweet.RetweetedStatusID) != v.retweeted_status_id {
t.Errorf("Expected retweeted status id %d, but got %d", v.retweeted_status_id, tweet.RetweetedStatusID) t.Errorf("Expected retweeted status id %d, but got %d", v.retweeted_status_id, tweet.RetweetedStatusID)

26
scraper/tweet.go
View File

@ -13,16 +13,17 @@ const DEFAULT_MAX_REPLIES_EAGER_LOAD = 50
type TweetID int64 type TweetID int64
type Tweet struct { type Tweet struct {
ID TweetID ID TweetID
UserID UserID UserID UserID
User *User User *User
Text string Text string
PostedAt time.Time PostedAt time.Time
NumLikes int NumLikes int
NumRetweets int NumRetweets int
NumReplies int NumReplies int
NumQuoteTweets int NumQuoteTweets int
InReplyTo TweetID InReplyToID TweetID
QuotedTweetID TweetID
Urls []Url Urls []Url
Images []Image Images []Image
@ -30,7 +31,6 @@ type Tweet struct {
Mentions []UserHandle Mentions []UserHandle
ReplyMentions []UserHandle ReplyMentions []UserHandle
Hashtags []string Hashtags []string
QuotedTweet TweetID
TombstoneType string TombstoneType string
IsStub bool IsStub bool
@ -95,7 +95,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
ret.NumRetweets = apiTweet.RetweetCount ret.NumRetweets = apiTweet.RetweetCount
ret.NumReplies = apiTweet.ReplyCount ret.NumReplies = apiTweet.ReplyCount
ret.NumQuoteTweets = apiTweet.QuoteCount ret.NumQuoteTweets = apiTweet.QuoteCount
ret.InReplyTo = TweetID(apiTweet.InReplyToStatusID) ret.InReplyToID = TweetID(apiTweet.InReplyToStatusID)
for _, url := range apiTweet.Entities.URLs { for _, url := range apiTweet.Entities.URLs {
var url_object Url var url_object Url
@ -132,7 +132,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
} }
} }
ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID) ret.QuotedTweetID = TweetID(apiTweet.QuotedStatusID)
for _, entity := range apiTweet.ExtendedEntities.Media { for _, entity := range apiTweet.ExtendedEntities.Media {
if entity.Type != "video" && entity.Type != "animated_gif" { if entity.Type != "video" && entity.Type != "animated_gif" {

12
scraper/tweet_test.go
View File

@ -48,8 +48,8 @@ func TestParseSingleTweet(t *testing.T) {
t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix()) t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix())
} }
if tweet.QuotedTweet != 0 { if tweet.QuotedTweetID != 0 {
t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweet) t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweetID)
} }
} }
@ -74,16 +74,16 @@ func TestParseTweetWithQuotedTweetAsLink(t *testing.T) {
} }
expected_replied_id := scraper.TweetID(1395882872729477131) expected_replied_id := scraper.TweetID(1395882872729477131)
if tweet.InReplyTo != expected_replied_id { if tweet.InReplyToID != expected_replied_id {
t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyTo) t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyToID)
} }
if len(tweet.ReplyMentions) != 0 { if len(tweet.ReplyMentions) != 0 {
t.Errorf("Wanted %v, got %v", []string{}, tweet.ReplyMentions) t.Errorf("Wanted %v, got %v", []string{}, tweet.ReplyMentions)
} }
expected_quoted_id := scraper.TweetID(1396194494710788100) expected_quoted_id := scraper.TweetID(1396194494710788100)
if tweet.QuotedTweet != expected_quoted_id { if tweet.QuotedTweetID != expected_quoted_id {
t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweet) t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweetID)
} }
} }