me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix spaces timestamps to remove the factor of 1000x

Browse Source
This commit is contained in:
Alessio 2023-08-19 22:35:10 -03:00
parent 7f24bb7893
commit 7a750877f6
7 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/TODO.txt
View File

@ -53,9 +53,6 @@ TODO post-tweets
TODO fetch-DMs TODO fetch-DMs
TODO spaces-timestamps-millis
- spaces timestamps are in milliseconds. Need to be converted to seconds
TODO fix-spaces-participants-duplicates TODO fix-spaces-participants-duplicates
- It is creating a new copy of the participants every time it gets scraped again - It is creating a new copy of the participants every time it gets scraped again

4
pkg/persistence/versions.go
View File

@ -118,6 +118,10 @@ var MIGRATIONS = []string{
);`, );`,
`create index if not exists index_tweets_posted_at on tweets (posted_at); `create index if not exists index_tweets_posted_at on tweets (posted_at);
create index if not exists index_retweets_retweeted_at on retweets (retweeted_at)`, create index if not exists index_retweets_retweeted_at on retweets (retweeted_at)`,
`update spaces set ended_at = ended_at/1000 where ended_at > strftime("%s")*500;
update spaces set updated_at = updated_at/1000 where updated_at > strftime("%s")*500;
update spaces set started_at = started_at/1000 where started_at > strftime("%s")*500;
update spaces set created_at = created_at/1000 where created_at > strftime("%s")*500;`,
} }
var ENGINE_DATABASE_VERSION = len(MIGRATIONS) var ENGINE_DATABASE_VERSION = len(MIGRATIONS)

8
pkg/scraper/api_types_spaces.go
View File

@ -64,10 +64,10 @@ func (r SpaceResponse) ToTweetTrove() TweetTrove {
space.Title = data.Metadata.Title space.Title = data.Metadata.Title
space.State = data.Metadata.State space.State = data.Metadata.State
space.CreatedById = UserID(data.Metadata.CreatorResults.Result.ID) space.CreatedById = UserID(data.Metadata.CreatorResults.Result.ID)
space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt) space.CreatedAt = TimestampFromUnix(data.Metadata.CreatedAt / 1000)
space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt) space.StartedAt = TimestampFromUnix(data.Metadata.StartedAt / 1000)
space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt) space.EndedAt = TimestampFromUnix(data.Metadata.EndedAt / 1000)
space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt) space.UpdatedAt = TimestampFromUnix(data.Metadata.UpdatedAt / 1000)
space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay space.IsAvailableForReplay = data.Metadata.IsSpaceAvailableForReplay
space.ReplayWatchCount = data.Metadata.TotalReplayWatched space.ReplayWatchCount = data.Metadata.TotalReplayWatched
space.LiveListenersCount = data.Metadata.TotalLiveListeners space.LiveListenersCount = data.Metadata.TotalLiveListeners

8
pkg/scraper/api_types_spaces_test.go
View File

@ -28,10 +28,10 @@ func TestParseSpaceResponse(t *testing.T) {
space := trove.Spaces["1BdxYypQzBgxX"] space := trove.Spaces["1BdxYypQzBgxX"]
assert.Equal(space.Title, "dreary weather 🌧️☔🌬️") assert.Equal(space.Title, "dreary weather 🌧️☔🌬️")
assert.Equal(space.CreatedById, UserID(1356335022815539201)) assert.Equal(space.CreatedById, UserID(1356335022815539201))
assert.Equal(int64(1665884387263), space.CreatedAt.Time.Unix()) assert.Equal(int64(1665884387), space.CreatedAt.Time.Unix())
assert.Equal(int64(1665884388222), space.StartedAt.Time.Unix()) assert.Equal(int64(1665884388), space.StartedAt.Time.Unix())
assert.Equal(int64(1665887491804), space.EndedAt.Time.Unix()) assert.Equal(int64(1665887491), space.EndedAt.Time.Unix())
assert.Equal(int64(1665887492705), space.UpdatedAt.Time.Unix()) assert.Equal(int64(1665887492), space.UpdatedAt.Time.Unix())
assert.False(space.IsAvailableForReplay) assert.False(space.IsAvailableForReplay)
assert.Equal(4, space.ReplayWatchCount) assert.Equal(4, space.ReplayWatchCount)
assert.Equal(1, space.LiveListenersCount) assert.Equal(1, space.LiveListenersCount)

2
pkg/scraper/space.go
View File

@ -27,7 +27,7 @@ type Space struct {
} }
func (space Space) FormatDuration() string { func (space Space) FormatDuration() string {
duration := space.EndedAt.Time.Sub(space.StartedAt.Time) / 1000 duration := space.EndedAt.Time.Sub(space.StartedAt.Time)
h := int(duration.Hours()) h := int(duration.Hours())
m := int(duration.Minutes()) % 60 m := int(duration.Minutes()) % 60
s := int(duration.Seconds()) % 60 s := int(duration.Seconds()) % 60

8
pkg/scraper/space_test.go
View File

@ -29,14 +29,14 @@ func TestParseSpace(t *testing.T) {
func TestFormatSpaceDuration(t *testing.T) { func TestFormatSpaceDuration(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
s := Space{ s := Space{
StartedAt: TimestampFromUnix(1000 * 1000), StartedAt: TimestampFromUnix(1000),
EndedAt: TimestampFromUnix(5000 * 1000), EndedAt: TimestampFromUnix(5000),
} }
assert.Equal(s.FormatDuration(), "1h06m") assert.Equal(s.FormatDuration(), "1h06m")
s.EndedAt = TimestampFromUnix(500000 * 1000) s.EndedAt = TimestampFromUnix(500000)
assert.Equal(s.FormatDuration(), "138h36m") assert.Equal(s.FormatDuration(), "138h36m")
s.EndedAt = TimestampFromUnix(1005 * 1000) s.EndedAt = TimestampFromUnix(1005)
assert.Equal(s.FormatDuration(), "0m05s") assert.Equal(s.FormatDuration(), "0m05s")
} }

2
sample_data/seed_data.sql
View File

@ -78,7 +78,7 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values
foreign key(created_by_id) references users(id) foreign key(created_by_id) references users(id)
); );
INSERT INTO spaces VALUES(323,'1OwGWwnoleRGQ',1178839081222115328,'https://t.co/kxr7O7hfJ6','Ended','I''m showering and the hot water ran out',1676225386889,1676225389824,1676235389824,1676229669381,1,11,255,1); INSERT INTO spaces VALUES(323,'1OwGWwnoleRGQ',1178839081222115328,'https://t.co/kxr7O7hfJ6','Ended','I''m showering and the hot water ran out',1676225386,1676225389,1676235389,1676229669,1,11,255,1);
CREATE TABLE tweets (rowid integer primary key, CREATE TABLE tweets (rowid integer primary key,