diff --git a/pkg/persistence/list_queries.go b/pkg/persistence/list_queries.go new file mode 100644 index 0000000..25c5a1b --- /dev/null +++ b/pkg/persistence/list_queries.go @@ -0,0 +1,101 @@ +package persistence + +import ( + "database/sql" + "errors" + "fmt" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +// Create an empty list, or rename an existing list +func (p Profile) SaveList(l *List) { + // Since the unique column is managed by the database (auto-increment) due to the existence of + // offline lists, we have to check for its existence first + var rowid ListID + if l.IsOnline { + // Online list; look up its rowid by its online ID + // TODO: maybe extract to a function + err := p.DB.Get(&rowid, "select rowid from lists where is_online = 1 and online_list_id = ?", l.ID) + if errors.Is(err, sql.ErrNoRows) { + // Doesn't exist yet + rowid = ListID(0) + } else if err != nil { + panic(err) + } + } else { + // For offline lists, just use the rowid + rowid = l.ID + } + + // If `rowid` is 0, then it doesn't exist yet; create it. Otherwise, update it + if rowid == ListID(0) { + result, err := p.DB.NamedExec(` + insert into lists (is_online, online_list_id, name) + values (:is_online, :online_list_id, :name) + `, l) + if err != nil { + panic(err) + } + id, err := result.LastInsertId() + if err != nil { + panic(err) + } + l.ID = ListID(id) + } else { + // Do update + _, err := p.DB.NamedExec(` + update lists set name = :name where rowid = :rowid + `, l) + if err != nil { + panic(err) + } + } +} + +func (p Profile) SaveListUsers(list_id ListID, trove TweetTrove) { + for user_id := range trove.Users { + p.SaveListUser(list_id, user_id) + } +} + +func (p Profile) SaveListUser(list_id ListID, user_id UserID) { + _, err := p.DB.Exec(`insert into list_users (list_id, user_id) values (?, ?) on conflict do nothing`, list_id, user_id) + if err != nil { + panic(fmt.Errorf("Error executing AddListUser(%d, %d):\n %w", list_id, user_id, err).Error()) + } +} + +func (p Profile) DeleteListUser(list_id ListID, user_id UserID) { + _, err := p.DB.Exec(`delete from list_users where list_id = ? and user_id = ?`, list_id, user_id) + if err != nil { + panic(fmt.Errorf("Error executing DeleteListUser(%d, %d):\n %w", list_id, user_id, err).Error()) + } +} + +func (p Profile) GetListById(list_id ListID) List { + var ret List + err := p.DB.Get(&ret, `select rowid, is_online, online_list_id, name from lists where rowid = ?`, list_id) + if err != nil { + panic(err) + } + return ret +} + +func (p Profile) GetListUsers(list_id ListID) []User { + var ret []User + err := p.DB.Select(&ret, ` + select `+USERS_ALL_SQL_FIELDS+` + from users + where id in (select user_id from list_users where list_id = ?) + `, list_id) + if err != nil { + panic(err) + } + return ret +} + +// XXX +// func (p Profile) GetFollowedUsers() List { +// err = +// } diff --git a/pkg/persistence/list_queries_test.go b/pkg/persistence/list_queries_test.go new file mode 100644 index 0000000..56e95a5 --- /dev/null +++ b/pkg/persistence/list_queries_test.go @@ -0,0 +1,154 @@ +package persistence_test + +import ( + "testing" + + "fmt" + "math/rand" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +func TestSaveAndLoadOfflineList(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Create an offline list + l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())} + require.Equal(l.ID, ListID(0)) + profile.SaveList(&l) + require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved + + // Check it comes back the same + new_l := profile.GetListById(l.ID) + assert.Equal(l.ID, new_l.ID) + assert.Equal(l.IsOnline, new_l.IsOnline) + assert.Equal(l.Name, new_l.Name) +} + +func TestRenameOfflineList(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Create an offline list + l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())} + profile.SaveList(&l) + require.NotEqual(l.ID, ListID(0)) + + // Rename it + l.Name = fmt.Sprintf("Untest List %d", rand.Int()) + profile.SaveList(&l) + + // Rename should be effective + new_l := profile.GetListById(l.ID) + assert.Equal(l.ID, new_l.ID) + assert.Equal(l.IsOnline, new_l.IsOnline) + assert.Equal(l.Name, new_l.Name) +} + +func TestSaveAndLoadOnlineList(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Create an online list + l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())} + require.Equal(l.ID, ListID(0)) + profile.SaveList(&l) + require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved + + // Check it comes back the same + new_l := profile.GetListById(l.ID) + assert.Equal(l.ID, new_l.ID) + assert.Equal(l.IsOnline, new_l.IsOnline) + assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists + assert.Equal(l.Name, new_l.Name) +} + +func TestRenameOnlineList(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Create an online list + l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())} + profile.SaveList(&l) + require.NotEqual(l.ID, ListID(0)) + + // Rename it + l.Name = fmt.Sprintf("Untest List %d", rand.Int()) + profile.SaveList(&l) + + // Rename should be effective + new_l := profile.GetListById(l.ID) + assert.Equal(l.ID, new_l.ID) + assert.Equal(l.IsOnline, new_l.IsOnline) + assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists + assert.Equal(l.Name, new_l.Name) +} + +func TestNoOnlineListWithoutID(t *testing.T) { + require := require.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Creating an online list with no OnlineID should fail + l := List{IsOnline: true, OnlineID: OnlineListID(0), Name: fmt.Sprintf("Test List %d", rand.Int())} + defer func() { + // Assert a panic occurred + r, is_ok := recover().(error) + require.True(is_ok) + require.Error(r) + }() + profile.SaveList(&l) +} + +func TestAddAndRemoveUserToList(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + // Create a list + l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())} + profile.SaveList(&l) + + // Check there's no users in it + require.Len(profile.GetListUsers(l.ID), 0) + + // Add a user to the list + u := create_dummy_user() + require.NoError(profile.SaveUser(&u)) + profile.SaveListUser(l.ID, u.ID) + + // Make sure it's in the list + users := profile.GetListUsers(l.ID) + require.Len(users, 1) + assert.Equal(u.Handle, users[0].Handle) + + // Addding it again should do nothing + profile.SaveListUser(l.ID, u.ID) + require.Len(profile.GetListUsers(l.ID), 1) + + // Remove the user from the list + profile.DeleteListUser(l.ID, u.ID) + + // Should be gone + require.Len(profile.GetListUsers(l.ID), 0) +} diff --git a/pkg/persistence/schema.sql b/pkg/persistence/schema.sql index 82be948..b5abf42 100644 --- a/pkg/persistence/schema.sql +++ b/pkg/persistence/schema.sql @@ -1,5 +1,9 @@ PRAGMA foreign_keys = on; + +-- Users +-- ----- + create table users (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), display_name text not null, @@ -25,6 +29,42 @@ create table users (rowid integer primary key, is_content_downloaded boolean default 0 ); +create table fake_user_sequence(latest_fake_id integer not null); +insert into fake_user_sequence values(0x4000000000000000); + +create table lists(rowid integer primary key, + is_online boolean not null default 0, + online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists + name text not null, + check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0)) + check (rowid != 0) +); +create table list_users(rowid integer primary key, + list_id integer not null, + user_id integer not null, + unique(list_id, user_id) + foreign key(list_id) references lists(rowid) + foreign key(user_id) references users(id) +); +create index if not exists index_list_users_list_id on list_users (list_id); +create index if not exists index_list_users_user_id on list_users (user_id); +insert into lists(rowid, name) values (1, "Offline Follows"); +insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1; + +create table follows(rowid integer primary key, + follower_id integer not null, + followee_id integer not null, + unique(follower_id, followee_id), + foreign key(follower_id) references users(id) + foreign key(followee_id) references users(id) +); +create index if not exists index_follows_followee_id on follows (followee_id); +create index if not exists index_follows_follower_id on follows (follower_id); + + +-- Tweets +-- ------ + create table tombstone_types (rowid integer primary key, short_name text not null unique, tombstone_text text not null unique @@ -39,7 +79,6 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter'), (8, 'newer-version-available', 'There’s a new version of this Tweet'); - create table tweets (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), user_id integer not null check(typeof(user_id) = 'integer'), @@ -69,15 +108,9 @@ create index if not exists index_tweets_in_reply_to_id on tweets (in_reply_to_id create index if not exists index_tweets_user_id on tweets (user_id); create index if not exists index_tweets_posted_at on tweets (posted_at); -create table retweets(rowid integer primary key, - retweet_id integer not null unique, - tweet_id integer not null, - retweeted_by integer not null, - retweeted_at integer not null, - foreign key(tweet_id) references tweets(id) - foreign key(retweeted_by) references users(id) -); -create index if not exists index_retweets_retweeted_at on retweets (retweeted_at); + +-- Tweet content +-- ------------- create table urls (rowid integer primary key, tweet_id integer not null, @@ -124,33 +157,6 @@ create table polls (rowid integer primary key, ); create index if not exists index_polls_tweet_id on polls (tweet_id); -create table spaces(rowid integer primary key, - id text unique not null, - created_by_id integer, - short_url text not null, - state text not null, - title text not null, - created_at integer not null, - started_at integer not null, - ended_at integer not null, - updated_at integer not null, - is_available_for_replay boolean not null, - replay_watch_count integer, - live_listeners_count integer, - is_details_fetched boolean not null default 0, - - foreign key(created_by_id) references users(id) -); - -create table space_participants(rowid integer primary key, - user_id integer not null, - space_id not null, - - unique(user_id, space_id) - foreign key(space_id) references spaces(id) - -- No foreign key for users, since they may not be downloaded yet and I don't want to - -- download every user who joins a space -); create table images (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), @@ -192,6 +198,56 @@ create table hashtags (rowid integer primary key, foreign key(tweet_id) references tweets(id) ); + +-- Retweets +-- -------- + +create table retweets(rowid integer primary key, + retweet_id integer not null unique, + tweet_id integer not null, + retweeted_by integer not null, + retweeted_at integer not null, + foreign key(tweet_id) references tweets(id) + foreign key(retweeted_by) references users(id) +); +create index if not exists index_retweets_retweeted_at on retweets (retweeted_at); + + +-- Spaces +-- ------ + +create table spaces(rowid integer primary key, + id text unique not null, + created_by_id integer, + short_url text not null, + state text not null, + title text not null, + created_at integer not null, + started_at integer not null, + ended_at integer not null, + updated_at integer not null, + is_available_for_replay boolean not null, + replay_watch_count integer, + live_listeners_count integer, + is_details_fetched boolean not null default 0, + + foreign key(created_by_id) references users(id) +); + +create table space_participants(rowid integer primary key, + user_id integer not null, + space_id not null, + + unique(user_id, space_id) + foreign key(space_id) references spaces(id) + -- No foreign key for users, since they may not be downloaded yet and I don't want to + -- download every user who joins a space +); + + +-- Likes +-- ----- + create table likes(rowid integer primary key, sort_order integer not null, -- Can't be unique because "-1" is used as "unknown" value user_id integer not null, @@ -204,19 +260,8 @@ create index if not exists index_likes_user_id on likes (user_id); create index if not exists index_likes_tweet_id on likes (tweet_id); -create table follows(rowid integer primary key, - follower_id integer not null, - followee_id integer not null, - unique(follower_id, followee_id), - foreign key(follower_id) references users(id) - foreign key(followee_id) references users(id) -); -create index if not exists index_follows_followee_id on follows (followee_id); -create index if not exists index_follows_follower_id on follows (follower_id); - - -create table fake_user_sequence(latest_fake_id integer not null); -insert into fake_user_sequence values(0x4000000000000000); +-- Direct Messages (DMs) +-- --------------------- create table chat_rooms (rowid integer primary key, id text unique not null, @@ -269,6 +314,10 @@ create table chat_message_reactions (rowid integer primary key, foreign key(sender_id) references users(id) ); + +-- Meta +-- ---- + create table database_version(rowid integer primary key, version_number integer not null unique ); diff --git a/pkg/scraper/list.go b/pkg/scraper/list.go index 64077b2..4fc471a 100644 --- a/pkg/scraper/list.go +++ b/pkg/scraper/list.go @@ -1,11 +1,13 @@ package scraper -type ListID int +type ListID int64 +type OnlineListID int64 type List struct { - ID ListID `db:"rowid"` - Type string `db:"type"` - Name string `db:"name"` + ID ListID `db:"rowid"` + IsOnline bool `db:"is_online"` + OnlineID OnlineListID `db:"online_list_id"` + Name string `db:"name"` UserIDs []UserID Users []*User diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index b4b91e9..f3ea2e6 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -55,6 +55,26 @@ INSERT INTO users VALUES (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116000,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0), (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one. Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522000,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0); +create table lists(rowid integer primary key, + is_online boolean not null default 0, + online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists + name text not null, + check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0)) + check (rowid != 0) +); +create table list_users(rowid integer primary key, + list_id integer not null, + user_id integer not null, + unique(list_id, user_id) + foreign key(list_id) references lists(rowid) + foreign key(user_id) references users(id) +); +create index if not exists index_list_users_list_id on list_users (list_id); +create index if not exists index_list_users_user_id on list_users (user_id); +insert into lists(rowid, name) values (1, "Offline Follows"); +insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1; + + create table tombstone_types (rowid integer primary key, short_name text not null unique, tombstone_text text not null unique