diff --git a/cmd/fetch_user_feed_to_stdout/main.go b/cmd/fetch_user_feed_to_stdout/main.go index 2dfb4fb..d5d4dbe 100644 --- a/cmd/fetch_user_feed_to_stdout/main.go +++ b/cmd/fetch_user_feed_to_stdout/main.go @@ -47,14 +47,14 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper. for i < len(tweets) && j < len(retweets) { if !tweets[i].PostedAt.Before(retweets[j].RetweetedAt) { tweet := tweets[i] - if tweet.User != user.ID { + if tweet.UserID != user.ID { i += 1 continue } - user, ok := users_dict[tweet.User] + user, ok := users_dict[tweet.UserID] if !ok { - log.Fatalf("User not found: %q", tweet.User) + log.Fatalf("User not found: %q", tweet.UserID) } print_tweet(tweets[i], user) @@ -69,9 +69,9 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper. if !ok { log.Fatalf("Tweet not found: %q", retweet.TweetID) } - original_poster, ok := users_dict[tweet.User] + original_poster, ok := users_dict[tweet.UserID] if !ok { - log.Fatalf("User not found: %q", tweet.User) + log.Fatalf("User not found: %q", tweet.UserID) } retweeter, ok := users_dict[retweet.RetweetedBy] if !ok { @@ -83,14 +83,14 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper. } for i < len(tweets) { tweet := tweets[i] - if tweet.User != user.ID { + if tweet.UserID != user.ID { i += 1 continue } - user, ok := users_dict[tweet.User] + user, ok := users_dict[tweet.UserID] if !ok { - log.Fatalf("User not found: %q", tweet.User) + log.Fatalf("User not found: %q", tweet.UserID) } print_tweet(tweets[i], user) @@ -106,9 +106,9 @@ func display_feed(user scraper.User, tweets []scraper.Tweet, retweets []scraper. if !ok { log.Fatalf("Tweet not found: %q", retweet.TweetID) } - original_poster, ok := users_dict[tweet.User] + original_poster, ok := users_dict[tweet.UserID] if !ok { - log.Fatalf("User not found: %q", tweet.User) + log.Fatalf("User not found: %q", tweet.UserID) } retweeter, ok := users_dict[retweet.RetweetedBy] if !ok { diff --git a/doc/curl requests b/doc/curl requests index 90c4048..297a4b8 100644 --- a/doc/curl requests +++ b/doc/curl requests @@ -21,6 +21,7 @@ curl \ # # A tweet and replies (conversation): +-----------------------------------NOTE: this one doesn't work, because it's missing a lot of GET params!----------------------------------- curl \ -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \ -H "X-Guest-Token: 1396177150890348547" \ diff --git a/persistence/schema.sql b/persistence/schema.sql index 1758a2a..86496e1 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -14,12 +14,14 @@ create table users (rowid integer primary key, is_verified boolean default 0, profile_image_url text, banner_image_url text, - pinned_tweet integer + pinned_tweet_id integer + + -- foreign key(pinned_tweet_id) references tweets(id) ); create table tweets (rowid integer primary key, id integer unique not null, - user integer not null, + user_id integer not null, text text not null, posted_at integer, num_likes integer, @@ -32,9 +34,9 @@ create table tweets (rowid integer primary key, mentions text, -- comma-separated hashtags text, -- comma-separated - foreign key(user) references users(id), - foreign key(in_reply_to) references tweets(id), - foreign key(quoted_tweet) references tweets(id) + foreign key(user_id) references users(id) + -- foreign key(in_reply_to) references tweets(id), + -- foreign key(quoted_tweet) references tweets(id) ); create table retweets(rowid integer primary key, diff --git a/scraper/retweet.go b/scraper/retweet.go index f90db4f..2d639d8 100644 --- a/scraper/retweet.go +++ b/scraper/retweet.go @@ -5,16 +5,18 @@ import ( ) type Retweet struct { - RetweetID TweetID - TweetID TweetID - RetweetedBy UserID - RetweetedAt time.Time + RetweetID TweetID + TweetID TweetID + Tweet *Tweet + RetweetedByID UserID + RetweetedBy *User + RetweetedAt time.Time } func ParseSingleRetweet(apiTweet APITweet) (ret Retweet, err error) { ret.RetweetID = TweetID(apiTweet.ID) ret.TweetID = TweetID(apiTweet.RetweetedStatusIDStr) - ret.RetweetedBy = UserID(apiTweet.UserIDStr) + ret.RetweetedByID = UserID(apiTweet.UserIDStr) ret.RetweetedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt) return } diff --git a/scraper/tweet.go b/scraper/tweet.go index 66a0c79..3964e68 100644 --- a/scraper/tweet.go +++ b/scraper/tweet.go @@ -11,7 +11,8 @@ type TweetID string type Tweet struct { ID TweetID - User UserID + UserID UserID + User *User Text string PostedAt time.Time NumLikes int @@ -32,7 +33,7 @@ func (t Tweet) String() string { return fmt.Sprintf( `ID %s, User %s: %q (%s). Likes: %d, Retweets: %d, QTs: %d, Replies: %d. Urls: %v Images: %v Mentions: %v Hashtags: %v`, - t.ID, t.User, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumQuoteTweets, t.NumReplies, t.Urls, t.Images, t.Mentions, t.Hashtags) + t.ID, t.UserID, t.Text, t.PostedAt, t.NumLikes, t.NumRetweets, t.NumQuoteTweets, t.NumReplies, t.Urls, t.Images, t.Mentions, t.Hashtags) } // Turn an APITweet, as returned from the scraper, into a properly structured Tweet object @@ -40,7 +41,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) { apiTweet.NormalizeContent() ret.ID = TweetID(apiTweet.ID) - ret.User = UserID(apiTweet.UserIDStr) + ret.UserID = UserID(apiTweet.UserIDStr) ret.Text = apiTweet.FullText ret.PostedAt, err = time.Parse(time.RubyDate, apiTweet.CreatedAt) diff --git a/scraper/user.go b/scraper/user.go index 049e64f..e95bf66 100644 --- a/scraper/user.go +++ b/scraper/user.go @@ -31,7 +31,8 @@ type User struct { IsVerified bool ProfileImageUrl string BannerImageUrl string - PinnedTweet TweetID + PinnedTweetID TweetID + PinnedTweet *Tweet } func (u User) String() string { @@ -59,7 +60,7 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) { ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS ret.BannerImageUrl = apiUser.ProfileBannerURL if len(apiUser.PinnedTweetIdsStr) > 0 { - ret.PinnedTweet = TweetID(apiUser.PinnedTweetIdsStr[0]) + ret.PinnedTweetID = TweetID(apiUser.PinnedTweetIdsStr[0]) } return } diff --git a/scraper/user_test.go b/scraper/user_test.go index 1b571f2..720e5a9 100644 --- a/scraper/user_test.go +++ b/scraper/user_test.go @@ -67,7 +67,7 @@ func TestParseSingleUser(t *testing.T) { if user.BannerImageUrl != expectedBannerImage { t.Errorf("Expected %q, got %q", expectedBannerImage, user.BannerImageUrl) } - if user.PinnedTweet != scraper.TweetID("1403835414373339136") { + if user.PinnedTweetID != scraper.TweetID("1403835414373339136") { t.Errorf("Expected %q, got %q", scraper.TweetID("1403835414373339136"), user.PinnedTweet) } }