me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: replace 'log.Debug(fmt.Sprintf(...))' with 'log.Debugf(...)' and remove 'scraper.' prefix in utils_test.go

Browse Source
This commit is contained in:
Alessio 2024-03-10 19:10:26 -07:00
parent f3b71a2633
commit 1ba4f91463
3 changed files with 109 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
pkg/persistence/utils_test.go
View File

@ -6,7 +6,7 @@ import (
"time"
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
)
// Load a test profile, or create it if it doesn't exist.
@ -43,32 +43,32 @@ func create_or_load_profile(profile_path string) persistence.Profile {
}
// Create a stable user with a fixed ID and handle
func create_stable_user() scraper.User {
return scraper.User{
ID: scraper.UserID(-1),
func create_stable_user() User {
return User{
ID: UserID(-1),
DisplayName: "stable display name",
Handle: scraper.UserHandle("handle stable"),
Handle: UserHandle("handle stable"),
Bio: "stable bio",
FollowersCount: 10,
FollowingCount: 2000,
Location: "stable location",
Website: "stable website",
JoinDate: scraper.TimestampFromUnix(10000000),
JoinDate: TimestampFromUnix(10000000),
IsVerified: true,
IsPrivate: false,
ProfileImageUrl: "stable profile image url",
ProfileImageLocalPath: "stable profile image local path",
BannerImageUrl: "stable banner image url",
BannerImageLocalPath: "stable image local path",
PinnedTweetID: scraper.TweetID(345),
PinnedTweetID: TweetID(345),
}
}
// Create a semi-stable Image based on the given ID
func create_image_from_id(id int) scraper.Image {
func create_image_from_id(id int) Image {
filename := fmt.Sprintf("image%d.jpg", id)
return scraper.Image{
ID: scraper.ImageID(id),
return Image{
ID: ImageID(id),
TweetID: -1,
Width: id * 10,
Height: id * 5,
@ -79,10 +79,10 @@ func create_image_from_id(id int) scraper.Image {
}
// Create a semi-stable Video based on the given ID
func create_video_from_id(id int) scraper.Video {
func create_video_from_id(id int) Video {
filename := fmt.Sprintf("video%d.jpg", id)
return scraper.Video{
ID: scraper.VideoID(id),
return Video{
ID: VideoID(id),
TweetID: -1,
Width: id * 10,
Height: id * 5,
@ -98,9 +98,9 @@ func create_video_from_id(id int) scraper.Video {
}
// Create a semi-stable Url based on the given ID
func create_url_from_id(id int) scraper.Url {
func create_url_from_id(id int) Url {
s := fmt.Sprint(id)
return scraper.Url{
return Url{
TweetID: -1,
Domain: s + "domain",
Text: s + "text",
@ -111,18 +111,18 @@ func create_url_from_id(id int) scraper.Url {
ThumbnailHeight: id * 7,
ThumbnailRemoteUrl: s + "remote url",
ThumbnailLocalPath: s + "local path",
CreatorID: scraper.UserID(id),
SiteID: scraper.UserID(id),
CreatorID: UserID(id),
SiteID: UserID(id),
HasCard: true,
IsContentDownloaded: false,
}
}
// Create a semi-stable Poll based on the given ID
func create_poll_from_id(id int) scraper.Poll {
func create_poll_from_id(id int) Poll {
s := fmt.Sprint(id)
return scraper.Poll{
ID: scraper.PollID(id),
return Poll{
ID: PollID(id),
TweetID: -1,
NumChoices: 2,
Choice1: s,
@ -130,86 +130,86 @@ func create_poll_from_id(id int) scraper.Poll {
Choice2: "Not " + s,
Choice2_Votes: 1500,
VotingDuration: 10,
VotingEndsAt: scraper.TimestampFromUnix(10000000),
LastUpdatedAt: scraper.TimestampFromUnix(10000),
VotingEndsAt: TimestampFromUnix(10000000),
LastUpdatedAt: TimestampFromUnix(10000),
}
}
// Create a stable tweet with a fixed ID and content
func create_stable_tweet() scraper.Tweet {
tweet_id := scraper.TweetID(-1)
return scraper.Tweet{
func create_stable_tweet() Tweet {
tweet_id := TweetID(-1)
return Tweet{
ID: tweet_id,
UserID: -1,
Text: "stable text",
PostedAt: scraper.TimestampFromUnix(10000000),
PostedAt: TimestampFromUnix(10000000),
NumLikes: 10,
NumRetweets: 10,
NumReplies: 10,
NumQuoteTweets: 10,
Videos: []scraper.Video{
Videos: []Video{
create_video_from_id(-1),
},
Urls: []scraper.Url{
Urls: []Url{
create_url_from_id(-1),
},
Images: []scraper.Image{
Images: []Image{
create_image_from_id(-1),
},
Mentions: scraper.CommaSeparatedList{},
Hashtags: scraper.CommaSeparatedList{},
Polls: []scraper.Poll{
Mentions: CommaSeparatedList{},
Hashtags: CommaSeparatedList{},
Polls: []Poll{
create_poll_from_id(-1),
},
Spaces: []scraper.Space{
Spaces: []Space{
create_space_from_id(-1),
},
SpaceID: scraper.SpaceID("some_id_-1"),
SpaceID: SpaceID("some_id_-1"),
IsConversationScraped: true,
LastScrapedAt: scraper.TimestampFromUnix(100000000),
LastScrapedAt: TimestampFromUnix(100000000),
}
}
// Create a stable retweet with a fixed ID and parameters
func create_stable_retweet() scraper.Retweet {
retweet_id := scraper.TweetID(-1)
return scraper.Retweet{
func create_stable_retweet() Retweet {
retweet_id := TweetID(-1)
return Retweet{
RetweetID: retweet_id,
TweetID: -1,
RetweetedByID: -1,
RetweetedAt: scraper.TimestampFromUnix(20000000),
RetweetedAt: TimestampFromUnix(20000000),
}
}
// Create a new user with a random ID and handle
func create_dummy_user() scraper.User {
func create_dummy_user() User {
rand.Seed(time.Now().UnixNano())
userID := rand.Int()
return scraper.User{
ID: scraper.UserID(userID),
return User{
ID: UserID(userID),
DisplayName: "display name",
Handle: scraper.UserHandle(fmt.Sprintf("handle%d", userID)),
Handle: UserHandle(fmt.Sprintf("handle%d", userID)),
Bio: "bio",
FollowersCount: 0,
FollowingCount: 1000,
Location: "location",
Website: "website",
JoinDate: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
JoinDate: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
IsVerified: false,
IsPrivate: true,
ProfileImageUrl: "profile image url",
ProfileImageLocalPath: "profile image local path",
BannerImageUrl: "banner image url",
BannerImageLocalPath: "banner image local path",
PinnedTweetID: scraper.TweetID(234),
PinnedTweetID: TweetID(234),
}
}
// Create a new tweet with a random ID and content
func create_dummy_tweet() scraper.Tweet {
func create_dummy_tweet() Tweet {
rand.Seed(time.Now().UnixNano())
tweet_id := scraper.TweetID(rand.Int())
tweet_id := TweetID(rand.Int())
img1 := create_image_from_id(rand.Int())
img1.TweetID = tweet_id
@ -229,94 +229,94 @@ func create_dummy_tweet() scraper.Tweet {
space := create_space_from_id(rand.Int())
space_id := space.ID
return scraper.Tweet{
return Tweet{
ID: tweet_id,
UserID: -1,
Text: "text",
PostedAt: scraper.Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
PostedAt: Timestamp{time.Now().Truncate(1e9)}, // Round to nearest second
NumLikes: 1,
NumRetweets: 2,
NumReplies: 3,
NumQuoteTweets: 4,
Videos: []scraper.Video{vid},
Urls: []scraper.Url{url1, url2},
Images: []scraper.Image{img1, img2},
Mentions: scraper.CommaSeparatedList{"mention1", "mention2"},
ReplyMentions: scraper.CommaSeparatedList{"replymention1", "replymention2"},
Hashtags: scraper.CommaSeparatedList{"hash1", "hash2"},
Polls: []scraper.Poll{poll},
Spaces: []scraper.Space{space},
Videos: []Video{vid},
Urls: []Url{url1, url2},
Images: []Image{img1, img2},
Mentions: CommaSeparatedList{"mention1", "mention2"},
ReplyMentions: CommaSeparatedList{"replymention1", "replymention2"},
Hashtags: CommaSeparatedList{"hash1", "hash2"},
Polls: []Poll{poll},
Spaces: []Space{space},
SpaceID: space_id,
}
}
// Create a random tombstone
func create_dummy_tombstone() scraper.Tweet {
func create_dummy_tombstone() Tweet {
rand.Seed(time.Now().UnixNano())
tweet_id := scraper.TweetID(rand.Int())
tweet_id := TweetID(rand.Int())
return scraper.Tweet{
return Tweet{
ID: tweet_id,
UserID: -1,
TombstoneType: "deleted",
IsStub: true,
Mentions: scraper.CommaSeparatedList{},
ReplyMentions: scraper.CommaSeparatedList{},
Hashtags: scraper.CommaSeparatedList{},
Spaces: []scraper.Space{},
Mentions: CommaSeparatedList{},
ReplyMentions: CommaSeparatedList{},
Hashtags: CommaSeparatedList{},
Spaces: []Space{},
}
}
// Create a new retweet with a random ID for a given TweetID
func create_dummy_retweet(tweet_id scraper.TweetID) scraper.Retweet {
func create_dummy_retweet(tweet_id TweetID) Retweet {
rand.Seed(time.Now().UnixNano())
retweet_id := scraper.TweetID(rand.Int())
retweet_id := TweetID(rand.Int())
return scraper.Retweet{
return Retweet{
RetweetID: retweet_id,
TweetID: tweet_id,
RetweetedByID: -1,
RetweetedAt: scraper.TimestampFromUnix(20000000),
RetweetedAt: TimestampFromUnix(20000000),
}
}
// Create a semi-stable Space given an ID
func create_space_from_id(id int) scraper.Space {
return scraper.Space{
ID: scraper.SpaceID(fmt.Sprintf("some_id_%d", id)),
func create_space_from_id(id int) Space {
return Space{
ID: SpaceID(fmt.Sprintf("some_id_%d", id)),
ShortUrl: fmt.Sprintf("short_url_%d", id),
State: "Running",
Title: "Some Title",
CreatedAt: scraper.TimestampFromUnix(1000),
StartedAt: scraper.TimestampFromUnix(2000),
EndedAt: scraper.TimestampFromUnix(3000),
UpdatedAt: scraper.TimestampFromUnix(4000),
CreatedAt: TimestampFromUnix(1000),
StartedAt: TimestampFromUnix(2000),
EndedAt: TimestampFromUnix(3000),
UpdatedAt: TimestampFromUnix(4000),
CreatedById: -1,
ParticipantIds: []scraper.UserID{-1},
ParticipantIds: []UserID{-1},
}
}
func create_dummy_like() scraper.Like {
return scraper.Like{
func create_dummy_like() Like {
return Like{
TweetID: create_stable_tweet().ID,
UserID: create_stable_user().ID,
SortID: scraper.LikeSortID(12345),
SortID: LikeSortID(12345),
}
}
func create_stable_chat_room() scraper.DMChatRoom {
id := scraper.DMChatRoomID("some chat room ID")
func create_stable_chat_room() DMChatRoom {
id := DMChatRoomID("some chat room ID")
return scraper.DMChatRoom{
return DMChatRoom{
ID: id,
Type: "ONE_ON_ONE",
LastMessagedAt: scraper.TimestampFromUnix(123),
LastMessagedAt: TimestampFromUnix(123),
IsNSFW: false,
Participants: map[scraper.UserID]scraper.DMChatParticipant{
scraper.UserID(-1): {
Participants: map[UserID]DMChatParticipant{
UserID(-1): {
DMChatRoomID: id,
UserID: scraper.UserID(-1),
LastReadEventID: scraper.DMMessageID(0),
UserID: UserID(-1),
LastReadEventID: DMMessageID(0),
IsChatSettingsValid: true,
IsNotificationsDisabled: false,
IsMentionNotificationsDisabled: false,
@ -329,20 +329,20 @@ func create_stable_chat_room() scraper.DMChatRoom {
}
}
func create_dummy_chat_room() scraper.DMChatRoom {
func create_dummy_chat_room() DMChatRoom {
rand.Seed(time.Now().UnixNano())
id := scraper.DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int()))
id := DMChatRoomID(fmt.Sprintf("Chat Room #%d", rand.Int()))
return scraper.DMChatRoom{
return DMChatRoom{
ID: id,
Type: "ONE_ON_ONE",
LastMessagedAt: scraper.TimestampFromUnix(10000),
LastMessagedAt: TimestampFromUnix(10000),
IsNSFW: false,
Participants: map[scraper.UserID]scraper.DMChatParticipant{
scraper.UserID(-1): {
Participants: map[UserID]DMChatParticipant{
UserID(-1): {
DMChatRoomID: id,
UserID: scraper.UserID(-1),
LastReadEventID: scraper.DMMessageID(0),
UserID: UserID(-1),
LastReadEventID: DMMessageID(0),
IsChatSettingsValid: true,
IsNotificationsDisabled: false,
IsMentionNotificationsDisabled: false,
@ -355,22 +355,22 @@ func create_dummy_chat_room() scraper.DMChatRoom {
}
}
func create_dummy_chat_message() scraper.DMMessage {
func create_dummy_chat_message() DMMessage {
rand.Seed(time.Now().UnixNano())
id := scraper.DMMessageID(rand.Int())
return scraper.DMMessage{
id := DMMessageID(rand.Int())
return DMMessage{
ID: id,
DMChatRoomID: create_stable_chat_room().ID,
SenderID: create_stable_user().ID,
SentAt: scraper.TimestampFromUnix(50000),
SentAt: TimestampFromUnix(50000),
RequestID: "fwjefkj",
Text: fmt.Sprintf("This is message #%d", id),
Reactions: map[scraper.UserID]scraper.DMReaction{
scraper.UserID(-1): {
Reactions: map[UserID]DMReaction{
UserID(-1): {
ID: id + 1,
DMMessageID: id,
SenderID: scraper.UserID(-1),
SentAt: scraper.TimestampFromUnix(51000),
SenderID: UserID(-1),
SentAt: TimestampFromUnix(51000),
Emoji: "🤔",
},
},

8
pkg/scraper/api_request_utils.go
View File

@ -151,9 +151,9 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
api.add_authentication_headers(req)
log.Debug(fmt.Sprintf("POST: %s\n", req.URL.String()))
log.Debugf("POST: %s\n", req.URL.String())
for header := range req.Header {
log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header)))
log.Debugf(" %s: %s\n", header, req.Header.Get(header))
}
log.Debug(" " + body)
@ -205,9 +205,9 @@ func (api *API) do_http(url string, cursor string, result interface{}) error {
api.add_authentication_headers(req)
log.Debug(fmt.Sprintf("GET: %s\n", req.URL.String()))
log.Debugf("GET: %s\n", req.URL.String())
for header := range req.Header {
log.Debug(fmt.Sprintf(" %s: %s\n", header, req.Header.Get(header)))
log.Debugf(" %s: %s\n", header, req.Header.Get(header))
}
resp, err := api.Client.Do(req)

4
pkg/scraper/api_types_v2.go
View File

@ -462,7 +462,7 @@ func (e *APIV2Entry) ParseID() (string, TweetID) {
func (e APIV2Entry) ToTweetTrove() TweetTrove {
defer func() {
if obj := recover(); obj != nil {
log.Warn(fmt.Sprintf("Panic while decoding entry: %s\n", e.OriginalJSON))
log.Warnf("Panic while decoding entry: %s\n", e.OriginalJSON)
panic(obj)
}
}()
@ -506,7 +506,7 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
} else if parts[0] == "whoToFollow" || parts[0] == "TopicsModule" || parts[0] == "tweetdetailrelatedtweets" {
// Ignore "Who to follow", "Topics" and "Related Tweets" modules.
// TODO: maybe we can capture these eventually
log.Debug(fmt.Sprintf("Skipping %s entry", e.EntryID))
log.Debugf("Skipping %s entry", e.EntryID)
} else {
log.Warn("TimelineTimelineModule with unknown EntryID: " + e.EntryID)
}