me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ID fields are now Hungarian-notated

Browse Source 
- tweet.User is now a pointer to a User object; tweet.UserID holds the user id
This commit is contained in:
Alessio 2021-07-24 10:19:05 -07:00
parent 741c508017
commit 96e2edfb8a
7 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/fetch_user_feed_to_stdout/main.go
View File

@ -47,14 +47,14 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper.
for i < len(tweets) && j < len(retweets) { for i < len(tweets) && j < len(retweets) {
if !tweets[i].PostedAt.Before(retweets[j].RetweetedAt) { if !tweets[i].PostedAt.Before(retweets[j].RetweetedAt) {
tweet := tweets[i] tweet := tweets[i]
if tweet.User != user.ID { if tweet.UserID != user.ID {
i += 1 i += 1
continue continue
} }
user, ok := users_dict[tweet.User] user, ok := users_dict[tweet.UserID]
if !ok { if !ok {
log.Fatalf("User not found: %q", tweet.User) log.Fatalf("User not found: %q", tweet.UserID)
} }
print_tweet(tweets[i], user) print_tweet(tweets[i], user)
@ -69,9 +69,9 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper.
if !ok { if !ok {
log.Fatalf("Tweet not found: %q", retweet.TweetID) log.Fatalf("Tweet not found: %q", retweet.TweetID)
} }
original_poster, ok := users_dict[tweet.User] original_poster, ok := users_dict[tweet.UserID]
if !ok { if !ok {
log.Fatalf("User not found: %q", tweet.User) log.Fatalf("User not found: %q", tweet.UserID)
} }
retweeter, ok := users_dict[retweet.RetweetedBy] retweeter, ok := users_dict[retweet.RetweetedBy]
if !ok { if !ok {
@ -83,14 +83,14 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper.
} }
for i < len(tweets) { for i < len(tweets) {
tweet := tweets[i] tweet := tweets[i]
if tweet.User != user.ID { if tweet.UserID != user.ID {
i += 1 i += 1
continue continue
} }
user, ok := users_dict[tweet.User] user, ok := users_dict[tweet.UserID]
if !ok { if !ok {
log.Fatalf("User not found: %q", tweet.User) log.Fatalf("User not found: %q", tweet.UserID)
} }
print_tweet(tweets[i], user) print_tweet(tweets[i], user)
@ -106,9 +106,9 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper.
if !ok { if !ok {
log.Fatalf("Tweet not found: %q", retweet.TweetID) log.Fatalf("Tweet not found: %q", retweet.TweetID)
} }
original_poster, ok := users_dict[tweet.User] original_poster, ok := users_dict[tweet.UserID]
if !ok { if !ok {
log.Fatalf("User not found: %q", tweet.User) log.Fatalf("User not found: %q", tweet.UserID)
} }
retweeter, ok := users_dict[retweet.RetweetedBy] retweeter, ok := users_dict[retweet.RetweetedBy]
if !ok { if !ok {

1
doc/curl requests
View File

@ -21,6 +21,7 @@ curl \
# #
# A tweet and replies (conversation): # A tweet and replies (conversation):
-----------------------------------NOTE: this one doesn't work, because it's missing a lot of GET params!-----------------------------------
curl \ curl \
-H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
-H "X-Guest-Token: 1396177150890348547" \ -H "X-Guest-Token: 1396177150890348547" \

12
persistence/schema.sql
View File

@ -14,12 +14,14 @@ create table users (rowid integer primary key,
is_verified boolean default 0, is_verified boolean default 0,
profile_image_url text, profile_image_url text,
banner_image_url text, banner_image_url text,
pinned_tweet integer pinned_tweet_id integer
-- foreign key(pinned_tweet_id) references tweets(id)
); );
create table tweets (rowid integer primary key, create table tweets (rowid integer primary key,
id integer unique not null, id integer unique not null,
user integer not null, user_id integer not null,
text text not null, text text not null,
posted_at integer, posted_at integer,
num_likes integer, num_likes integer,
@ -32,9 +34,9 @@ create table tweets (rowid integer primary key,
mentions text, -- comma-separated mentions text, -- comma-separated
hashtags text, -- comma-separated hashtags text, -- comma-separated
foreign key(user) references users(id), foreign key(user_id) references users(id)
foreign key(in_reply_to) references tweets(id), -- foreign key(in_reply_to) references tweets(id),
foreign key(quoted_tweet) references tweets(id) -- foreign key(quoted_tweet) references tweets(id)
); );
create table retweets(rowid integer primary key, create table retweets(rowid integer primary key,

6
scraper/retweet.go
View File

@ -7,14 +7,16 @@ import (
type Retweet struct { type Retweet struct {
RetweetID TweetID RetweetID TweetID
TweetID TweetID TweetID TweetID
RetweetedBy UserID Tweet *Tweet
RetweetedByID UserID
RetweetedBy *User
RetweetedAt time.Time RetweetedAt time.Time
} }
func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) { func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) {
ret.RetweetID = TweetID(apiTweet.ID) ret.RetweetID = TweetID(apiTweet.ID)
ret.TweetID = TweetID(apiTweet.RetweetedStatusIDStr) ret.TweetID = TweetID(apiTweet.RetweetedStatusIDStr)
ret.RetweetedBy = UserID(apiTweet.UserIDStr) ret.RetweetedByID = UserID(apiTweet.UserIDStr)
ret.RetweetedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt) ret.RetweetedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt)
return return
} }

7
scraper/tweet.go
View File

@ -11,7 +11,8 @@ type TweetID string
type Tweet struct { type Tweet struct {
ID TweetID ID TweetID
User UserID UserID UserID
User *User
Text string Text string
PostedAt time.Time PostedAt time.Time
NumLikes int NumLikes int
@ -32,7 +33,7 @@ func (t Tweet) String() string {
return fmt.Sprintf( return fmt.Sprintf(
`ID %s, User %s: %q (%s). Likes: %d, Retweets: %d, QTs: %d, Replies: %d. `ID %s, User %s: %q (%s). Likes: %d, Retweets: %d, QTs: %d, Replies: %d.
Urls: %v Images: %v Mentions: %v Hashtags: %v`, Urls: %v Images: %v Mentions: %v Hashtags: %v`,
t.ID, t.User, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumQuoteTweets, t.NumReplies, t.Urls, t.Images, t.Mentions, t.Hashtags) t.ID, t.UserID, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumQuoteTweets, t.NumReplies, t.Urls, t.Images, t.Mentions, t.Hashtags)
} }
// Turn an APITweet, as returned from the scraper, into a properly structured Tweet object // Turn an APITweet, as returned from the scraper, into a properly structured Tweet object
@ -40,7 +41,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
apiTweet.NormalizeContent() apiTweet.NormalizeContent()
ret.ID = TweetID(apiTweet.ID) ret.ID = TweetID(apiTweet.ID)
ret.User = UserID(apiTweet.UserIDStr) ret.UserID = UserID(apiTweet.UserIDStr)
ret.Text = apiTweet.FullText ret.Text = apiTweet.FullText
ret.PostedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt) ret.PostedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt)

5
scraper/user.go
View File

@ -31,7 +31,8 @@ type User struct {
IsVerified bool IsVerified bool
ProfileImageUrl string ProfileImageUrl string
BannerImageUrl string BannerImageUrl string
PinnedTweet TweetID PinnedTweetID TweetID
PinnedTweet *Tweet
} }
func (u User) String() string { func (u User) String() string {
@ -59,7 +60,7 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) {
ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS
ret.BannerImageUrl = apiUser.ProfileBannerURL ret.BannerImageUrl = apiUser.ProfileBannerURL
if len(apiUser.PinnedTweetIdsStr) > 0 { if len(apiUser.PinnedTweetIdsStr) > 0 {
ret.PinnedTweet = TweetID(apiUser.PinnedTweetIdsStr[0]) ret.PinnedTweetID = TweetID(apiUser.PinnedTweetIdsStr[0])
} }
return return
} }

2
scraper/user_test.go
View File

@ -67,7 +67,7 @@ func TestParseSingleUser(t *testing.T) {
if user.BannerImageUrl != expectedBannerImage { if user.BannerImageUrl != expectedBannerImage {
t.Errorf("Expected %q, got %q", expectedBannerImage, user.BannerImageUrl) t.Errorf("Expected %q, got %q", expectedBannerImage, user.BannerImageUrl)
} }
if user.PinnedTweet != scraper.TweetID("1403835414373339136") { if user.PinnedTweetID != scraper.TweetID("1403835414373339136") {
t.Errorf("Expected %q, got %q", scraper.TweetID("1403835414373339136"), user.PinnedTweet) t.Errorf("Expected %q, got %q", scraper.TweetID("1403835414373339136"), user.PinnedTweet)
} }
} }