diff --git a/pkg/persistence/utils_test.go b/pkg/persistence/utils_test.go index fdd6ede..579cffa 100644 --- a/pkg/persistence/utils_test.go +++ b/pkg/persistence/utils_test.go @@ -6,7 +6,7 @@ import ( "time" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Load a test profile, or create it if it doesn't exist. @@ -43,32 +43,32 @@ func create_or_load_profile(profile_path string) persistence.Profile { } // Create a stable user with a fixed ID and handle -func create_stable_user() scraper.User { - return scraper.User{ - ID: scraper.UserID(-1), +func create_stable_user() User { + return User{ + ID: UserID(-1), DisplayName: "stable display name", - Handle: scraper.UserHandle("handle stable"), + Handle: UserHandle("handle stable"), Bio: "stable bio", FollowersCount: 10, FollowingCount: 2000, Location: "stable location", Website: "stable website", - JoinDate: scraper.TimestampFromUnix(10000000), + JoinDate: TimestampFromUnix(10000000), IsVerified: true, IsPrivate: false, ProfileImageUrl: "stable profile image url", ProfileImageLocalPath: "stable profile image local path", BannerImageUrl: "stable banner image url", BannerImageLocalPath: "stable image local path", - PinnedTweetID: scraper.TweetID(345), + PinnedTweetID: TweetID(345), } } // Create a semi-stable Image based on the given ID -func create_image_from_id(id int) scraper.Image { +func create_image_from_id(id int) Image { filename := fmt.Sprintf("image%d.jpg", id) - return scraper.Image{ - ID: scraper.ImageID(id), + return Image{ + ID: ImageID(id), TweetID: -1, Width: id * 10, Height: id * 5, @@ -79,10 +79,10 @@ func create_image_from_id(id int) scraper.Image { } // Create a semi-stable Video based on the given ID -func create_video_from_id(id int) scraper.Video { +func create_video_from_id(id int) Video { filename := fmt.Sprintf("video%d.jpg", id) - return scraper.Video{ - ID: scraper.VideoID(id), + return Video{ + ID: VideoID(id), TweetID: -1, Width: id * 10, Height: id * 5, @@ -98,9 +98,9 @@ func create_video_from_id(id int) scraper.Video { } // Create a semi-stable Url based on the given ID -func create_url_from_id(id int) scraper.Url { +func create_url_from_id(id int) Url { s := fmt.Sprint(id) - return scraper.Url{ + return Url{ TweetID: -1, Domain: s + "domain", Text: s + "text", @@ -111,18 +111,18 @@ func create_url_from_id(id int) scraper.Url { ThumbnailHeight: id * 7, ThumbnailRemoteUrl: s + "remote url", ThumbnailLocalPath: s + "local path", - CreatorID: scraper.UserID(id), - SiteID: scraper.UserID(id), + CreatorID: UserID(id), + SiteID: UserID(id), HasCard: true, IsContentDownloaded: false, } } // Create a semi-stable Poll based on the given ID -func create_poll_from_id(id int) scraper.Poll { +func create_poll_from_id(id int) Poll { s := fmt.Sprint(id) - return scraper.Poll{ - ID: scraper.PollID(id), + return Poll{ + ID: PollID(id), TweetID: -1, NumChoices: 2, Choice1: s, @@ -130,86 +130,86 @@ func create_poll_from_id(id int) scraper.Poll { Choice2: "Not " + s, Choice2_Votes: 1500, VotingDuration: 10, - VotingEndsAt: scraper.TimestampFromUnix(10000000), - LastUpdatedAt: scraper.TimestampFromUnix(10000), + VotingEndsAt: TimestampFromUnix(10000000), + LastUpdatedAt: TimestampFromUnix(10000), } } // Create a stable tweet with a fixed ID and content -func create_stable_tweet() scraper.Tweet { - tweet_id := scraper.TweetID(-1) - return scraper.Tweet{ +func create_stable_tweet() Tweet { + tweet_id := TweetID(-1) + return Tweet{ ID: tweet_id, UserID: -1, Text: "stable text", - PostedAt: scraper.TimestampFromUnix(10000000), + PostedAt: TimestampFromUnix(10000000), NumLikes: 10, NumRetweets: 10, NumReplies: 10, NumQuoteTweets: 10, - Videos: []scraper.Video{ + Videos: []Video{ create_video_from_id(-1), }, - Urls: []scraper.Url{ + Urls: []Url{ create_url_from_id(-1), }, - Images: []scraper.Image{ + Images: []Image{ create_image_from_id(-1), }, - Mentions: scraper.CommaSeparatedList{}, - Hashtags: scraper.CommaSeparatedList{}, - Polls: []scraper.Poll{ + Mentions: CommaSeparatedList{}, + Hashtags: CommaSeparatedList{}, + Polls: []Poll{ create_poll_from_id(-1), }, - Spaces: []scraper.Space{ + Spaces: []Space{ create_space_from_id(-1), }, - SpaceID: scraper.SpaceID("some_id_-1"), + SpaceID: SpaceID("some_id_-1"), IsConversationScraped: true, - LastScrapedAt: scraper.TimestampFromUnix(100000000), + LastScrapedAt: TimestampFromUnix(100000000), } } // Create a stable retweet with a fixed ID and parameters -func create_stable_retweet() scraper.Retweet { - retweet_id := scraper.TweetID(-1) - return scraper.Retweet{ +func create_stable_retweet() Retweet { + retweet_id := TweetID(-1) + return Retweet{ RetweetID: retweet_id, TweetID: -1, RetweetedByID: -1, - RetweetedAt: scraper.TimestampFromUnix(20000000), + RetweetedAt: TimestampFromUnix(20000000), } } // Create a new user with a random ID and handle -func create_dummy_user() scraper.User { +func create_dummy_user() User { rand.Seed(time.Now().UnixNano()) userID := rand.Int() - return scraper.User{ - ID: scraper.UserID(userID), + return User{ + ID: UserID(userID), DisplayName: "display name", - Handle: scraper.UserHandle(fmt.Sprintf("handle%d", userID)), + Handle: UserHandle(fmt.Sprintf("handle%d", userID)), Bio: "bio", FollowersCount: 0, FollowingCount: 1000, Location: "location", Website: "website", - JoinDate: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second + JoinDate: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second IsVerified: false, IsPrivate: true, ProfileImageUrl: "profile image url", ProfileImageLocalPath: "profile image local path", BannerImageUrl: "banner image url", BannerImageLocalPath: "banner image local path", - PinnedTweetID: scraper.TweetID(234), + PinnedTweetID: TweetID(234), } } // Create a new tweet with a random ID and content -func create_dummy_tweet() scraper.Tweet { +func create_dummy_tweet() Tweet { rand.Seed(time.Now().UnixNano()) - tweet_id := scraper.TweetID(rand.Int()) + tweet_id := TweetID(rand.Int()) img1 := create_image_from_id(rand.Int()) img1.TweetID = tweet_id @@ -229,94 +229,94 @@ func create_dummy_tweet() scraper.Tweet { space := create_space_from_id(rand.Int()) space_id := space.ID - return scraper.Tweet{ + return Tweet{ ID: tweet_id, UserID: -1, Text: "text", - PostedAt: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second + PostedAt: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second NumLikes: 1, NumRetweets: 2, NumReplies: 3, NumQuoteTweets: 4, - Videos: []scraper.Video{vid}, - Urls: []scraper.Url{url1, url2}, - Images: []scraper.Image{img1, img2}, - Mentions: scraper.CommaSeparatedList{"mention1", "mention2"}, - ReplyMentions: scraper.CommaSeparatedList{"replymention1", "replymention2"}, - Hashtags: scraper.CommaSeparatedList{"hash1", "hash2"}, - Polls: []scraper.Poll{poll}, - Spaces: []scraper.Space{space}, + Videos: []Video{vid}, + Urls: []Url{url1, url2}, + Images: []Image{img1, img2}, + Mentions: CommaSeparatedList{"mention1", "mention2"}, + ReplyMentions: CommaSeparatedList{"replymention1", "replymention2"}, + Hashtags: CommaSeparatedList{"hash1", "hash2"}, + Polls: []Poll{poll}, + Spaces: []Space{space}, SpaceID: space_id, } } // Create a random tombstone -func create_dummy_tombstone() scraper.Tweet { +func create_dummy_tombstone() Tweet { rand.Seed(time.Now().UnixNano()) - tweet_id := scraper.TweetID(rand.Int()) + tweet_id := TweetID(rand.Int()) - return scraper.Tweet{ + return Tweet{ ID: tweet_id, UserID: -1, TombstoneType: "deleted", IsStub: true, - Mentions: scraper.CommaSeparatedList{}, - ReplyMentions: scraper.CommaSeparatedList{}, - Hashtags: scraper.CommaSeparatedList{}, - Spaces: []scraper.Space{}, + Mentions: CommaSeparatedList{}, + ReplyMentions: CommaSeparatedList{}, + Hashtags: CommaSeparatedList{}, + Spaces: []Space{}, } } // Create a new retweet with a random ID for a given TweetID -func create_dummy_retweet(tweet_id scraper.TweetID) scraper.Retweet { +func create_dummy_retweet(tweet_id TweetID) Retweet { rand.Seed(time.Now().UnixNano()) - retweet_id := scraper.TweetID(rand.Int()) + retweet_id := TweetID(rand.Int()) - return scraper.Retweet{ + return Retweet{ RetweetID: retweet_id, TweetID: tweet_id, RetweetedByID: -1, - RetweetedAt: scraper.TimestampFromUnix(20000000), + RetweetedAt: TimestampFromUnix(20000000), } } // Create a semi-stable Space given an ID -func create_space_from_id(id int) scraper.Space { - return scraper.Space{ - ID: scraper.SpaceID(fmt.Sprintf("some_id_%d", id)), +func create_space_from_id(id int) Space { + return Space{ + ID: SpaceID(fmt.Sprintf("some_id_%d", id)), ShortUrl: fmt.Sprintf("short_url_%d", id), State: "Running", Title: "Some Title", - CreatedAt: scraper.TimestampFromUnix(1000), - StartedAt: scraper.TimestampFromUnix(2000), - EndedAt: scraper.TimestampFromUnix(3000), - UpdatedAt: scraper.TimestampFromUnix(4000), + CreatedAt: TimestampFromUnix(1000), + StartedAt: TimestampFromUnix(2000), + EndedAt: TimestampFromUnix(3000), + UpdatedAt: TimestampFromUnix(4000), CreatedById: -1, - ParticipantIds: []scraper.UserID{-1}, + ParticipantIds: []UserID{-1}, } } -func create_dummy_like() scraper.Like { - return scraper.Like{ +func create_dummy_like() Like { + return Like{ TweetID: create_stable_tweet().ID, UserID: create_stable_user().ID, - SortID: scraper.LikeSortID(12345), + SortID: LikeSortID(12345), } } -func create_stable_chat_room() scraper.DMChatRoom { - id := scraper.DMChatRoomID("some chat room ID") +func create_stable_chat_room() DMChatRoom { + id := DMChatRoomID("some chat room ID") - return scraper.DMChatRoom{ + return DMChatRoom{ ID: id, Type: "ONE_ON_ONE", - LastMessagedAt: scraper.TimestampFromUnix(123), + LastMessagedAt: TimestampFromUnix(123), IsNSFW: false, - Participants: map[scraper.UserID]scraper.DMChatParticipant{ - scraper.UserID(-1): { + Participants: map[UserID]DMChatParticipant{ + UserID(-1): { DMChatRoomID: id, - UserID: scraper.UserID(-1), - LastReadEventID: scraper.DMMessageID(0), + UserID: UserID(-1), + LastReadEventID: DMMessageID(0), IsChatSettingsValid: true, IsNotificationsDisabled: false, IsMentionNotificationsDisabled: false, @@ -329,20 +329,20 @@ func create_stable_chat_room() scraper.DMChatRoom { } } -func create_dummy_chat_room() scraper.DMChatRoom { +func create_dummy_chat_room() DMChatRoom { rand.Seed(time.Now().UnixNano()) - id := scraper.DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int())) + id := DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int())) - return scraper.DMChatRoom{ + return DMChatRoom{ ID: id, Type: "ONE_ON_ONE", - LastMessagedAt: scraper.TimestampFromUnix(10000), + LastMessagedAt: TimestampFromUnix(10000), IsNSFW: false, - Participants: map[scraper.UserID]scraper.DMChatParticipant{ - scraper.UserID(-1): { + Participants: map[UserID]DMChatParticipant{ + UserID(-1): { DMChatRoomID: id, - UserID: scraper.UserID(-1), - LastReadEventID: scraper.DMMessageID(0), + UserID: UserID(-1), + LastReadEventID: DMMessageID(0), IsChatSettingsValid: true, IsNotificationsDisabled: false, IsMentionNotificationsDisabled: false, @@ -355,22 +355,22 @@ func create_dummy_chat_room() scraper.DMChatRoom { } } -func create_dummy_chat_message() scraper.DMMessage { +func create_dummy_chat_message() DMMessage { rand.Seed(time.Now().UnixNano()) - id := scraper.DMMessageID(rand.Int()) - return scraper.DMMessage{ + id := DMMessageID(rand.Int()) + return DMMessage{ ID: id, DMChatRoomID: create_stable_chat_room().ID, SenderID: create_stable_user().ID, - SentAt: scraper.TimestampFromUnix(50000), + SentAt: TimestampFromUnix(50000), RequestID: "fwjefkj", Text: fmt.Sprintf("This is message #%d", id), - Reactions: map[scraper.UserID]scraper.DMReaction{ - scraper.UserID(-1): { + Reactions: map[UserID]DMReaction{ + UserID(-1): { ID: id + 1, DMMessageID: id, - SenderID: scraper.UserID(-1), - SentAt: scraper.TimestampFromUnix(51000), + SenderID: UserID(-1), + SentAt: TimestampFromUnix(51000), Emoji: "🤔", }, }, diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go index fe17ab7..4c2ef73 100644 --- a/pkg/scraper/api_request_utils.go +++ b/pkg/scraper/api_request_utils.go @@ -151,9 +151,9 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error api.add_authentication_headers(req) - log.Debug(fmt.Sprintf("POST: %s\n", req.URL.String())) + log.Debugf("POST: %s\n", req.URL.String()) for header := range req.Header { - log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header))) + log.Debugf(" %s: %s\n", header, req.Header.Get(header)) } log.Debug(" " + body) @@ -205,9 +205,9 @@ func (api *API) do_http(url string, cursor string, result interface{}) error { api.add_authentication_headers(req) - log.Debug(fmt.Sprintf("GET: %s\n", req.URL.String())) + log.Debugf("GET: %s\n", req.URL.String()) for header := range req.Header { - log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header))) + log.Debugf(" %s: %s\n", header, req.Header.Get(header)) } resp, err := api.Client.Do(req) diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index b0760b1..d33a13a 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -462,7 +462,7 @@ func (e *APIV2Entry) ParseID() (string, TweetID) { func (e APIV2Entry) ToTweetTrove() TweetTrove { defer func() { if obj := recover(); obj != nil { - log.Warn(fmt.Sprintf("Panic while decoding entry: %s\n", e.OriginalJSON)) + log.Warnf("Panic while decoding entry: %s\n", e.OriginalJSON) panic(obj) } }() @@ -506,7 +506,7 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove { } else if parts[0] == "whoToFollow" || parts[0] == "TopicsModule" || parts[0] == "tweetdetailrelatedtweets" { // Ignore "Who to follow", "Topics" and "Related Tweets" modules. // TODO: maybe we can capture these eventually - log.Debug(fmt.Sprintf("Skipping %s entry", e.EntryID)) + log.Debugf("Skipping %s entry", e.EntryID) } else { log.Warn("TimelineTimelineModule with unknown EntryID: " + e.EntryID) }