me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: replace 'log.Debug(fmt.Sprintf(...))' with 'log.Debugf(...)' and remove 'scraper.' prefix in utils_test.go

Browse Source
This commit is contained in:
Alessio 2024-03-10 19:10:26 -07:00
parent f3b71a2633
commit 1ba4f91463
3 changed files with 109 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
pkg/persistence/utils_test.go
View File

@ -6,7 +6,7 @@ import (
"time" "time"
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
) )
// Load a test profile, or create it if it doesn't exist. // Load a test profile, or create it if it doesn't exist.
@ -43,32 +43,32 @@ func create_or_load_profile(profile_path string) persistence.Profile {
} }
// Create a stable user with a fixed ID and handle // Create a stable user with a fixed ID and handle
func create_stable_user() scraper.User { func create_stable_user() User {
return scraper.User{ return User{
ID: scraper.UserID(-1), ID: UserID(-1),
DisplayName: "stable display name", DisplayName: "stable display name",
Handle: scraper.UserHandle("handle stable"), Handle: UserHandle("handle stable"),
Bio: "stable bio", Bio: "stable bio",
FollowersCount: 10, FollowersCount: 10,
FollowingCount: 2000, FollowingCount: 2000,
Location: "stable location", Location: "stable location",
Website: "stable website", Website: "stable website",
JoinDate: scraper.TimestampFromUnix(10000000), JoinDate: TimestampFromUnix(10000000),
IsVerified: true, IsVerified: true,
IsPrivate: false, IsPrivate: false,
ProfileImageUrl: "stable profile image url", ProfileImageUrl: "stable profile image url",
ProfileImageLocalPath: "stable profile image local path", ProfileImageLocalPath: "stable profile image local path",
BannerImageUrl: "stable banner image url", BannerImageUrl: "stable banner image url",
BannerImageLocalPath: "stable image local path", BannerImageLocalPath: "stable image local path",
PinnedTweetID: scraper.TweetID(345), PinnedTweetID: TweetID(345),
} }
} }
// Create a semi-stable Image based on the given ID // Create a semi-stable Image based on the given ID
func create_image_from_id(id int) scraper.Image { func create_image_from_id(id int) Image {
filename := fmt.Sprintf("image%d.jpg", id) filename := fmt.Sprintf("image%d.jpg", id)
return scraper.Image{ return Image{
ID: scraper.ImageID(id), ID: ImageID(id),
TweetID: -1, TweetID: -1,
Width: id * 10, Width: id * 10,
Height: id * 5, Height: id * 5,
@ -79,10 +79,10 @@ func create_image_from_id(id int) scraper.Image {
} }
// Create a semi-stable Video based on the given ID // Create a semi-stable Video based on the given ID
func create_video_from_id(id int) scraper.Video { func create_video_from_id(id int) Video {
filename := fmt.Sprintf("video%d.jpg", id) filename := fmt.Sprintf("video%d.jpg", id)
return scraper.Video{ return Video{
ID: scraper.VideoID(id), ID: VideoID(id),
TweetID: -1, TweetID: -1,
Width: id * 10, Width: id * 10,
Height: id * 5, Height: id * 5,
@ -98,9 +98,9 @@ func create_video_from_id(id int) scraper.Video {
} }
// Create a semi-stable Url based on the given ID // Create a semi-stable Url based on the given ID
func create_url_from_id(id int) scraper.Url { func create_url_from_id(id int) Url {
s := fmt.Sprint(id) s := fmt.Sprint(id)
return scraper.Url{ return Url{
TweetID: -1, TweetID: -1,
Domain: s + "domain", Domain: s + "domain",
Text: s + "text", Text: s + "text",
@ -111,18 +111,18 @@ func create_url_from_id(id int) scraper.Url {
ThumbnailHeight: id * 7, ThumbnailHeight: id * 7,
ThumbnailRemoteUrl: s + "remote url", ThumbnailRemoteUrl: s + "remote url",
ThumbnailLocalPath: s + "local path", ThumbnailLocalPath: s + "local path",
CreatorID: scraper.UserID(id), CreatorID: UserID(id),
SiteID: scraper.UserID(id), SiteID: UserID(id),
HasCard: true, HasCard: true,
IsContentDownloaded: false, IsContentDownloaded: false,
} }
} }
// Create a semi-stable Poll based on the given ID // Create a semi-stable Poll based on the given ID
func create_poll_from_id(id int) scraper.Poll { func create_poll_from_id(id int) Poll {
s := fmt.Sprint(id) s := fmt.Sprint(id)
return scraper.Poll{ return Poll{
ID: scraper.PollID(id), ID: PollID(id),
TweetID: -1, TweetID: -1,
NumChoices: 2, NumChoices: 2,
Choice1: s, Choice1: s,
@ -130,86 +130,86 @@ func create_poll_from_id(id int) scraper.Poll {
Choice2: "Not " + s, Choice2: "Not " + s,
Choice2_Votes: 1500, Choice2_Votes: 1500,
VotingDuration: 10, VotingDuration: 10,
VotingEndsAt: scraper.TimestampFromUnix(10000000), VotingEndsAt: TimestampFromUnix(10000000),
LastUpdatedAt: scraper.TimestampFromUnix(10000), LastUpdatedAt: TimestampFromUnix(10000),
} }
} }
// Create a stable tweet with a fixed ID and content // Create a stable tweet with a fixed ID and content
func create_stable_tweet() scraper.Tweet { func create_stable_tweet() Tweet {
tweet_id := scraper.TweetID(-1) tweet_id := TweetID(-1)
return scraper.Tweet{ return Tweet{
ID: tweet_id, ID: tweet_id,
UserID: -1, UserID: -1,
Text: "stable text", Text: "stable text",
PostedAt: scraper.TimestampFromUnix(10000000), PostedAt: TimestampFromUnix(10000000),
NumLikes: 10, NumLikes: 10,
NumRetweets: 10, NumRetweets: 10,
NumReplies: 10, NumReplies: 10,
NumQuoteTweets: 10, NumQuoteTweets: 10,
Videos: []scraper.Video{ Videos: []Video{
create_video_from_id(-1), create_video_from_id(-1),
}, },
Urls: []scraper.Url{ Urls: []Url{
create_url_from_id(-1), create_url_from_id(-1),
}, },
Images: []scraper.Image{ Images: []Image{
create_image_from_id(-1), create_image_from_id(-1),
}, },
Mentions: scraper.CommaSeparatedList{}, Mentions: CommaSeparatedList{},
Hashtags: scraper.CommaSeparatedList{}, Hashtags: CommaSeparatedList{},
Polls: []scraper.Poll{ Polls: []Poll{
create_poll_from_id(-1), create_poll_from_id(-1),
}, },
Spaces: []scraper.Space{ Spaces: []Space{
create_space_from_id(-1), create_space_from_id(-1),
}, },
SpaceID: scraper.SpaceID("some_id_-1"), SpaceID: SpaceID("some_id_-1"),
IsConversationScraped: true, IsConversationScraped: true,
LastScrapedAt: scraper.TimestampFromUnix(100000000), LastScrapedAt: TimestampFromUnix(100000000),
} }
} }
// Create a stable retweet with a fixed ID and parameters // Create a stable retweet with a fixed ID and parameters
func create_stable_retweet() scraper.Retweet { func create_stable_retweet() Retweet {
retweet_id := scraper.TweetID(-1) retweet_id := TweetID(-1)
return scraper.Retweet{ return Retweet{
RetweetID: retweet_id, RetweetID: retweet_id,
TweetID: -1, TweetID: -1,
RetweetedByID: -1, RetweetedByID: -1,
RetweetedAt: scraper.TimestampFromUnix(20000000), RetweetedAt: TimestampFromUnix(20000000),
} }
} }
// Create a new user with a random ID and handle // Create a new user with a random ID and handle
func create_dummy_user() scraper.User { func create_dummy_user() User {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
userID := rand.Int() userID := rand.Int()
return scraper.User{ return User{
ID: scraper.UserID(userID), ID: UserID(userID),
DisplayName: "display name", DisplayName: "display name",
Handle: scraper.UserHandle(fmt.Sprintf("handle%d", userID)), Handle: UserHandle(fmt.Sprintf("handle%d", userID)),
Bio: "bio", Bio: "bio",
FollowersCount: 0, FollowersCount: 0,
FollowingCount: 1000, FollowingCount: 1000,
Location: "location", Location: "location",
Website: "website", Website: "website",
JoinDate: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second JoinDate: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
IsVerified: false, IsVerified: false,
IsPrivate: true, IsPrivate: true,
ProfileImageUrl: "profile image url", ProfileImageUrl: "profile image url",
ProfileImageLocalPath: "profile image local path", ProfileImageLocalPath: "profile image local path",
BannerImageUrl: "banner image url", BannerImageUrl: "banner image url",
BannerImageLocalPath: "banner image local path", BannerImageLocalPath: "banner image local path",
PinnedTweetID: scraper.TweetID(234), PinnedTweetID: TweetID(234),
} }
} }
// Create a new tweet with a random ID and content // Create a new tweet with a random ID and content
func create_dummy_tweet() scraper.Tweet { func create_dummy_tweet() Tweet {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
tweet_id := scraper.TweetID(rand.Int()) tweet_id := TweetID(rand.Int())
img1 := create_image_from_id(rand.Int()) img1 := create_image_from_id(rand.Int())
img1.TweetID = tweet_id img1.TweetID = tweet_id
@ -229,94 +229,94 @@ func create_dummy_tweet() scraper.Tweet {
space := create_space_from_id(rand.Int()) space := create_space_from_id(rand.Int())
space_id := space.ID space_id := space.ID
return scraper.Tweet{ return Tweet{
ID: tweet_id, ID: tweet_id,
UserID: -1, UserID: -1,
Text: "text", Text: "text",
PostedAt: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second PostedAt: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
NumLikes: 1, NumLikes: 1,
NumRetweets: 2, NumRetweets: 2,
NumReplies: 3, NumReplies: 3,
NumQuoteTweets: 4, NumQuoteTweets: 4,
Videos: []scraper.Video{vid}, Videos: []Video{vid},
Urls: []scraper.Url{url1, url2}, Urls: []Url{url1, url2},
Images: []scraper.Image{img1, img2}, Images: []Image{img1, img2},
Mentions: scraper.CommaSeparatedList{"mention1", "mention2"}, Mentions: CommaSeparatedList{"mention1", "mention2"},
ReplyMentions: scraper.CommaSeparatedList{"replymention1", "replymention2"}, ReplyMentions: CommaSeparatedList{"replymention1", "replymention2"},
Hashtags: scraper.CommaSeparatedList{"hash1", "hash2"}, Hashtags: CommaSeparatedList{"hash1", "hash2"},
Polls: []scraper.Poll{poll}, Polls: []Poll{poll},
Spaces: []scraper.Space{space}, Spaces: []Space{space},
SpaceID: space_id, SpaceID: space_id,
} }
} }
// Create a random tombstone // Create a random tombstone
func create_dummy_tombstone() scraper.Tweet { func create_dummy_tombstone() Tweet {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
tweet_id := scraper.TweetID(rand.Int()) tweet_id := TweetID(rand.Int())
return scraper.Tweet{ return Tweet{
ID: tweet_id, ID: tweet_id,
UserID: -1, UserID: -1,
TombstoneType: "deleted", TombstoneType: "deleted",
IsStub: true, IsStub: true,
Mentions: scraper.CommaSeparatedList{}, Mentions: CommaSeparatedList{},
ReplyMentions: scraper.CommaSeparatedList{}, ReplyMentions: CommaSeparatedList{},
Hashtags: scraper.CommaSeparatedList{}, Hashtags: CommaSeparatedList{},
Spaces: []scraper.Space{}, Spaces: []Space{},
} }
} }
// Create a new retweet with a random ID for a given TweetID // Create a new retweet with a random ID for a given TweetID
func create_dummy_retweet(tweet_id scraper.TweetID) scraper.Retweet { func create_dummy_retweet(tweet_id TweetID) Retweet {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
retweet_id := scraper.TweetID(rand.Int()) retweet_id := TweetID(rand.Int())
return scraper.Retweet{ return Retweet{
RetweetID: retweet_id, RetweetID: retweet_id,
TweetID: tweet_id, TweetID: tweet_id,
RetweetedByID: -1, RetweetedByID: -1,
RetweetedAt: scraper.TimestampFromUnix(20000000), RetweetedAt: TimestampFromUnix(20000000),
} }
} }
// Create a semi-stable Space given an ID // Create a semi-stable Space given an ID
func create_space_from_id(id int) scraper.Space { func create_space_from_id(id int) Space {
return scraper.Space{ return Space{
ID: scraper.SpaceID(fmt.Sprintf("some_id_%d", id)), ID: SpaceID(fmt.Sprintf("some_id_%d", id)),
ShortUrl: fmt.Sprintf("short_url_%d", id), ShortUrl: fmt.Sprintf("short_url_%d", id),
State: "Running", State: "Running",
Title: "Some Title", Title: "Some Title",
CreatedAt: scraper.TimestampFromUnix(1000), CreatedAt: TimestampFromUnix(1000),
StartedAt: scraper.TimestampFromUnix(2000), StartedAt: TimestampFromUnix(2000),
EndedAt: scraper.TimestampFromUnix(3000), EndedAt: TimestampFromUnix(3000),
UpdatedAt: scraper.TimestampFromUnix(4000), UpdatedAt: TimestampFromUnix(4000),
CreatedById: -1, CreatedById: -1,
ParticipantIds: []scraper.UserID{-1}, ParticipantIds: []UserID{-1},
} }
} }
func create_dummy_like() scraper.Like { func create_dummy_like() Like {
return scraper.Like{ return Like{
TweetID: create_stable_tweet().ID, TweetID: create_stable_tweet().ID,
UserID: create_stable_user().ID, UserID: create_stable_user().ID,
SortID: scraper.LikeSortID(12345), SortID: LikeSortID(12345),
} }
} }
func create_stable_chat_room() scraper.DMChatRoom { func create_stable_chat_room() DMChatRoom {
id := scraper.DMChatRoomID("some chat room ID") id := DMChatRoomID("some chat room ID")
return scraper.DMChatRoom{ return DMChatRoom{
ID: id, ID: id,
Type: "ONE_ON_ONE", Type: "ONE_ON_ONE",
LastMessagedAt: scraper.TimestampFromUnix(123), LastMessagedAt: TimestampFromUnix(123),
IsNSFW: false, IsNSFW: false,
Participants: map[scraper.UserID]scraper.DMChatParticipant{ Participants: map[UserID]DMChatParticipant{
scraper.UserID(-1): { UserID(-1): {
DMChatRoomID: id, DMChatRoomID: id,
UserID: scraper.UserID(-1), UserID: UserID(-1),
LastReadEventID: scraper.DMMessageID(0), LastReadEventID: DMMessageID(0),
IsChatSettingsValid: true, IsChatSettingsValid: true,
IsNotificationsDisabled: false, IsNotificationsDisabled: false,
IsMentionNotificationsDisabled: false, IsMentionNotificationsDisabled: false,
@ -329,20 +329,20 @@ func create_stable_chat_room() scraper.DMChatRoom {
} }
} }
func create_dummy_chat_room() scraper.DMChatRoom { func create_dummy_chat_room() DMChatRoom {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
id := scraper.DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int())) id := DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int()))
return scraper.DMChatRoom{ return DMChatRoom{
ID: id, ID: id,
Type: "ONE_ON_ONE", Type: "ONE_ON_ONE",
LastMessagedAt: scraper.TimestampFromUnix(10000), LastMessagedAt: TimestampFromUnix(10000),
IsNSFW: false, IsNSFW: false,
Participants: map[scraper.UserID]scraper.DMChatParticipant{ Participants: map[UserID]DMChatParticipant{
scraper.UserID(-1): { UserID(-1): {
DMChatRoomID: id, DMChatRoomID: id,
UserID: scraper.UserID(-1), UserID: UserID(-1),
LastReadEventID: scraper.DMMessageID(0), LastReadEventID: DMMessageID(0),
IsChatSettingsValid: true, IsChatSettingsValid: true,
IsNotificationsDisabled: false, IsNotificationsDisabled: false,
IsMentionNotificationsDisabled: false, IsMentionNotificationsDisabled: false,
@ -355,22 +355,22 @@ func create_dummy_chat_room() scraper.DMChatRoom {
} }
} }
func create_dummy_chat_message() scraper.DMMessage { func create_dummy_chat_message() DMMessage {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
id := scraper.DMMessageID(rand.Int()) id := DMMessageID(rand.Int())
return scraper.DMMessage{ return DMMessage{
ID: id, ID: id,
DMChatRoomID: create_stable_chat_room().ID, DMChatRoomID: create_stable_chat_room().ID,
SenderID: create_stable_user().ID, SenderID: create_stable_user().ID,
SentAt: scraper.TimestampFromUnix(50000), SentAt: TimestampFromUnix(50000),
RequestID: "fwjefkj", RequestID: "fwjefkj",
Text: fmt.Sprintf("This is message #%d", id), Text: fmt.Sprintf("This is message #%d", id),
Reactions: map[scraper.UserID]scraper.DMReaction{ Reactions: map[UserID]DMReaction{
scraper.UserID(-1): { UserID(-1): {
ID: id + 1, ID: id + 1,
DMMessageID: id, DMMessageID: id,
SenderID: scraper.UserID(-1), SenderID: UserID(-1),
SentAt: scraper.TimestampFromUnix(51000), SentAt: TimestampFromUnix(51000),
Emoji: "🤔", Emoji: "🤔",
}, },
}, },

8
pkg/scraper/api_request_utils.go
View File

@ -151,9 +151,9 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
api.add_authentication_headers(req) api.add_authentication_headers(req)
log.Debug(fmt.Sprintf("POST: %s\n", req.URL.String())) log.Debugf("POST: %s\n", req.URL.String())
for header := range req.Header { for header := range req.Header {
log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header))) log.Debugf(" %s: %s\n", header, req.Header.Get(header))
} }
log.Debug(" " + body) log.Debug(" " + body)
@ -205,9 +205,9 @@ func (api *API) do_http(url string, cursor string, result interface{}) error {
api.add_authentication_headers(req) api.add_authentication_headers(req)
log.Debug(fmt.Sprintf("GET: %s\n", req.URL.String())) log.Debugf("GET: %s\n", req.URL.String())
for header := range req.Header { for header := range req.Header {
log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header))) log.Debugf(" %s: %s\n", header, req.Header.Get(header))
} }
resp, err := api.Client.Do(req) resp, err := api.Client.Do(req)

4
pkg/scraper/api_types_v2.go
View File

@ -462,7 +462,7 @@ func (e *APIV2Entry) ParseID() (string, TweetID) {
func (e APIV2Entry) ToTweetTrove() TweetTrove { func (e APIV2Entry) ToTweetTrove() TweetTrove {
defer func() { defer func() {
if obj := recover(); obj != nil { if obj := recover(); obj != nil {
log.Warn(fmt.Sprintf("Panic while decoding entry: %s\n", e.OriginalJSON)) log.Warnf("Panic while decoding entry: %s\n", e.OriginalJSON)
panic(obj) panic(obj)
} }
}() }()
@ -506,7 +506,7 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
} else if parts[0] == "whoToFollow" || parts[0] == "TopicsModule" || parts[0] == "tweetdetailrelatedtweets" { } else if parts[0] == "whoToFollow" || parts[0] == "TopicsModule" || parts[0] == "tweetdetailrelatedtweets" {
// Ignore "Who to follow", "Topics" and "Related Tweets" modules. // Ignore "Who to follow", "Topics" and "Related Tweets" modules.
// TODO: maybe we can capture these eventually // TODO: maybe we can capture these eventually
log.Debug(fmt.Sprintf("Skipping %s entry", e.EntryID)) log.Debugf("Skipping %s entry", e.EntryID)
} else { } else {
log.Warn("TimelineTimelineModule with unknown EntryID: " + e.EntryID) log.Warn("TimelineTimelineModule with unknown EntryID: " + e.EntryID)
} }