From 7a750877f601b59a5058e7346f6763256c41e409 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sat, 19 Aug 2023 22:35:10 -0300 Subject: [PATCH] Fix spaces timestamps to remove the factor of 1000x --- doc/TODO.txt | 3 --- pkg/persistence/versions.go | 4 ++++ pkg/scraper/api_types_spaces.go | 8 ++++---- pkg/scraper/api_types_spaces_test.go | 8 ++++---- pkg/scraper/space.go | 2 +- pkg/scraper/space_test.go | 8 ++++---- sample_data/seed_data.sql | 2 +- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/doc/TODO.txt b/doc/TODO.txt index 1cc35b2..1e4e125 100644 --- a/doc/TODO.txt +++ b/doc/TODO.txt @@ -53,9 +53,6 @@ TODO post-tweets TODO fetch-DMs -TODO spaces-timestamps-millis -- spaces timestamps are in milliseconds. Need to be converted to seconds - TODO fix-spaces-participants-duplicates - It is creating a new copy of the participants every time it gets scraped again diff --git a/pkg/persistence/versions.go b/pkg/persistence/versions.go index bc2a3ad..3b9df86 100644 --- a/pkg/persistence/versions.go +++ b/pkg/persistence/versions.go @@ -118,6 +118,10 @@ var MIGRATIONS = []string{ );`, `create index if not exists index_tweets_posted_at on tweets (posted_at); create index if not exists index_retweets_retweeted_at on retweets (retweeted_at)`, + `update spaces set ended_at = ended_at/1000 where ended_at > strftime("%s")*500; + update spaces set updated_at = updated_at/1000 where updated_at > strftime("%s")*500; + update spaces set started_at = started_at/1000 where started_at > strftime("%s")*500; + update spaces set created_at = created_at/1000 where created_at > strftime("%s")*500;`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) diff --git a/pkg/scraper/api_types_spaces.go b/pkg/scraper/api_types_spaces.go index 2c015ed..aad0292 100644 --- a/pkg/scraper/api_types_spaces.go +++ b/pkg/scraper/api_types_spaces.go @@ -64,10 +64,10 @@ func (r SpaceResponse) ToTweetTrove() TweetTrove { space.Title = data.Metadata.Title space.State = data.Metadata.State space.CreatedById = UserID(data.Metadata.CreatorResults.Result.ID) - space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt) - space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt) - space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt) - space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt) + space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt / 1000) + space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt / 1000) + space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt / 1000) + space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt / 1000) space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay space.ReplayWatchCount = data.Metadata.TotalReplayWatched space.LiveListenersCount = data.Metadata.TotalLiveListeners diff --git a/pkg/scraper/api_types_spaces_test.go b/pkg/scraper/api_types_spaces_test.go index 7974fe3..49b85f7 100644 --- a/pkg/scraper/api_types_spaces_test.go +++ b/pkg/scraper/api_types_spaces_test.go @@ -28,10 +28,10 @@ func TestParseSpaceResponse(t *testing.T) { space := trove.Spaces["1BdxYypQzBgxX"] assert.Equal(space.Title, "dreary weather 🌧️☔🌬️") assert.Equal(space.CreatedById, UserID(1356335022815539201)) - assert.Equal(int64(1665884387263), space.CreatedAt.Time.Unix()) - assert.Equal(int64(1665884388222), space.StartedAt.Time.Unix()) - assert.Equal(int64(1665887491804), space.EndedAt.Time.Unix()) - assert.Equal(int64(1665887492705), space.UpdatedAt.Time.Unix()) + assert.Equal(int64(1665884387), space.CreatedAt.Time.Unix()) + assert.Equal(int64(1665884388), space.StartedAt.Time.Unix()) + assert.Equal(int64(1665887491), space.EndedAt.Time.Unix()) + assert.Equal(int64(1665887492), space.UpdatedAt.Time.Unix()) assert.False(space.IsAvailableForReplay) assert.Equal(4, space.ReplayWatchCount) assert.Equal(1, space.LiveListenersCount) diff --git a/pkg/scraper/space.go b/pkg/scraper/space.go index 4e7c1e4..b08c3ba 100644 --- a/pkg/scraper/space.go +++ b/pkg/scraper/space.go @@ -27,7 +27,7 @@ type Space struct { } func (space Space) FormatDuration() string { - duration := space.EndedAt.Time.Sub(space.StartedAt.Time) / 1000 + duration := space.EndedAt.Time.Sub(space.StartedAt.Time) h := int(duration.Hours()) m := int(duration.Minutes()) % 60 s := int(duration.Seconds()) % 60 diff --git a/pkg/scraper/space_test.go b/pkg/scraper/space_test.go index d178e9e..8f51e27 100644 --- a/pkg/scraper/space_test.go +++ b/pkg/scraper/space_test.go @@ -29,14 +29,14 @@ func TestParseSpace(t *testing.T) { func TestFormatSpaceDuration(t *testing.T) { assert := assert.New(t) s := Space{ - StartedAt: TimestampFromUnix(1000 * 1000), - EndedAt: TimestampFromUnix(5000 * 1000), + StartedAt: TimestampFromUnix(1000), + EndedAt: TimestampFromUnix(5000), } assert.Equal(s.FormatDuration(), "1h06m") - s.EndedAt = TimestampFromUnix(500000 * 1000) + s.EndedAt = TimestampFromUnix(500000) assert.Equal(s.FormatDuration(), "138h36m") - s.EndedAt = TimestampFromUnix(1005 * 1000) + s.EndedAt = TimestampFromUnix(1005) assert.Equal(s.FormatDuration(), "0m05s") } diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index 717d822..7e5a08c 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -78,7 +78,7 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values foreign key(created_by_id) references users(id) ); -INSERT INTO spaces VALUES(323,'1OwGWwnoleRGQ',1178839081222115328,'https://t.co/kxr7O7hfJ6','Ended','I''m showering and the hot water ran out',1676225386889,1676225389824,1676235389824,1676229669381,1,11,255,1); +INSERT INTO spaces VALUES(323,'1OwGWwnoleRGQ',1178839081222115328,'https://t.co/kxr7O7hfJ6','Ended','I''m showering and the hot water ran out',1676225386,1676225389,1676235389,1676229669,1,11,255,1); CREATE TABLE tweets (rowid integer primary key,