package persistence import ( "fmt" sql "github.com/jmoiron/sqlx" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/terminal_utils" ) type VersionMismatchError struct { EngineVersion int DatabaseVersion int } func (e VersionMismatchError) Error() string { return fmt.Sprintf( `This profile was created with database schema version %d, which is newer than this application's database schema version, %d. Please upgrade this application to a newer version to use this profile. Or downgrade the profile's schema version, somehow.`, e.DatabaseVersion, e.EngineVersion, ) } // The Nth entry is the migration that moves you from version N to version N+1. var MIGRATIONS = []string{ `create table polls (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), tweet_id integer not null, num_choices integer not null, choice1 text, choice1_votes integer, choice2 text, choice2_votes integer, choice3 text, choice3_votes integer, choice4 text, choice4_votes integer, voting_duration integer not null, -- in seconds voting_ends_at integer not null, last_scraped_at integer not null, foreign key(tweet_id) references tweets(id) );`, `alter table tweets add column is_conversation_scraped boolean default 0; alter table tweets add column last_scraped_at integer not null default 0`, `update tombstone_types set tombstone_text = 'This Tweet is from a suspended account' where rowid = 2; insert into tombstone_types (rowid, short_name, tombstone_text) values (5, 'violated', 'This Tweet violated the Twitter Rules'), (6, 'no longer exists', 'This Tweet is from an account that no longer exists')`, `alter table videos add column thumbnail_remote_url text not null default "missing"; alter table videos add column thumbnail_local_filename text not null default "missing"`, // 5 `alter table videos add column duration integer not null default 0; alter table videos add column view_count integer not null default 0`, `alter table users add column is_banned boolean default 0`, `alter table urls add column short_text text not null default ""`, `insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. ' || 'This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`, `alter table users add column is_followed boolean default 0`, // 10 `create table fake_user_sequence(latest_fake_id integer not null); insert into fake_user_sequence values(0x4000000000000000); alter table users add column is_id_fake boolean default 0;`, `delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like 'https://twitter.com/%/status/' || tweets.quoted_tweet_id || "%")`, `create table spaces(rowid integer primary key, id text unique not null, short_url text not null ); alter table tweets add column space_id text references spaces(id)`, `alter table videos add column is_blocked_by_dmca boolean not null default 0`, `create index if not exists index_tweets_in_reply_to_id on tweets (in_reply_to_id); create index if not exists index_urls_tweet_id on urls (tweet_id); create index if not exists index_polls_tweet_id on polls (tweet_id); create index if not exists index_images_tweet_id on images (tweet_id); create index if not exists index_videos_tweet_id on videos (tweet_id);`, // 15 `alter table spaces add column created_by_id integer references users(id); alter table spaces add column state text not null default ""; alter table spaces add column title text not null default ""; alter table spaces add column created_at integer; alter table spaces add column started_at integer; alter table spaces add column ended_at integer; alter table spaces add column updated_at integer; alter table spaces add column is_available_for_replay boolean not null default 0; alter table spaces add column replay_watch_count integer; alter table spaces add column live_listeners_count integer; alter table spaces add column is_details_fetched boolean not null default 0; create table space_participants(rowid integer primary key, user_id integer not null, space_id not null, foreign key(space_id) references spaces(id) );`, `create index if not exists index_tweets_user_id on tweets (user_id);`, `alter table tweets add column is_expandable bool not null default 0;`, `create table space_participants_uniq(rowid integer primary key, user_id integer not null, space_id not null, unique(user_id, space_id) foreign key(space_id) references spaces(id) -- No foreign key for users, since they may not be downloaded yet and I don't want to -- download every user who joins a space ); insert or replace into space_participants_uniq(rowid, user_id, space_id) select rowid, user_id, space_id from space_participants; drop table space_participants; alter table space_participants_uniq rename to space_participants; vacuum;`, `create table likes(rowid integer primary key, sort_order integer unique not null, user_id integer not null, tweet_id integer not null, unique(user_id, tweet_id) foreign key(user_id) references users(id) foreign key(tweet_id) references tweets(id) );`, // 20 `create index if not exists index_tweets_posted_at on tweets (posted_at); create index if not exists index_retweets_retweeted_at on retweets (retweeted_at)`, `update spaces set ended_at = ended_at/1000 where ended_at > strftime("%s")*500; update spaces set updated_at = updated_at/1000 where updated_at > strftime("%s")*500; update spaces set started_at = started_at/1000 where started_at > strftime("%s")*500; update spaces set created_at = created_at/1000 where created_at > strftime("%s")*500;`, `alter table users add column is_deleted boolean default 0`, `begin transaction; alter table likes rename to likes_old; create table likes(rowid integer primary key, sort_order integer not null, user_id integer not null, tweet_id integer not null, unique(user_id, tweet_id) foreign key(user_id) references users(id) foreign key(tweet_id) references tweets(id) ); create index if not exists index_likes_user_id on likes (user_id); create index if not exists index_likes_tweet_id on likes (tweet_id); insert into likes select * from likes_old; drop table likes_old; commit; vacuum;`, `insert into tombstone_types(rowid, short_name, tombstone_text) values (8, 'newer-version-available', 'There’s a new version of this Tweet')`, // 25 `create table chat_rooms (rowid integer primary key, id text unique not null, type text not null, last_messaged_at integer not null, is_nsfw boolean not null, -- Group DM info created_at integer not null, created_by_user_id integer not null, name text not null default '', avatar_image_remote_url text not null default '', avatar_image_local_path text not null default '' ); create table chat_room_participants(rowid integer primary key, chat_room_id text not null, user_id integer not null, last_read_event_id integer not null, is_chat_settings_valid boolean not null default 0, is_notifications_disabled boolean not null, is_mention_notifications_disabled boolean not null, is_read_only boolean not null, is_trusted boolean not null, is_muted boolean not null, status text not null, unique(chat_room_id, user_id) ); create table chat_messages (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), chat_room_id text not null, sender_id integer not null, sent_at integer not null, request_id text not null, in_reply_to_id integer, text text not null, embedded_tweet_id integer not null default 0, foreign key(chat_room_id) references chat_rooms(id) foreign key(sender_id) references users(id) ); create table chat_message_reactions (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), message_id integer not null, sender_id integer not null, sent_at integer not null, emoji text not null, foreign key(message_id) references chat_messages(id) foreign key(sender_id) references users(id) );`, `create table follows(rowid integer primary key, follower_id integer not null, followee_id integer not null, unique(follower_id, followee_id), foreign key(follower_id) references users(id) foreign key(followee_id) references users(id) ); create index if not exists index_follows_followee_id on follows (followee_id); create index if not exists index_follows_follower_id on follows (follower_id);`, `update tweets set posted_at = posted_at * 1000, last_scraped_at = last_scraped_at * 1000; update users set join_date = join_date * 1000; update spaces set created_at = created_at * 1000, started_at = started_at * 1000, ended_at = ended_at * 1000, updated_at = updated_at * 1000; update retweets set retweeted_at = retweeted_at * 1000; update polls set voting_ends_at = voting_ends_at * 1000, last_scraped_at = last_scraped_at * 1000; update chat_rooms set created_at = created_at * 1000;`, `create table lists(rowid integer primary key, is_online boolean not null default 0, online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists name text not null, check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0)) check (rowid != 0) ); create table list_users(rowid integer primary key, list_id integer not null, user_id integer not null, unique(list_id, user_id) foreign key(list_id) references lists(rowid) on delete cascade foreign key(user_id) references users(id) ); create index if not exists index_list_users_list_id on list_users (list_id); create index if not exists index_list_users_user_id on list_users (user_id); insert into lists(rowid, name) values (1, "Offline Follows"); insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;`, `create table chat_message_images (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), chat_message_id integer not null, width integer not null, height integer not null, remote_url text not null unique, local_filename text not null unique, is_downloaded boolean default 0, foreign key(chat_message_id) references chat_messages(id) ); create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id); create table chat_message_videos (rowid integer primary key, id integer unique not null check(typeof(id) = 'integer'), chat_message_id integer not null, width integer not null, height integer not null, remote_url text not null unique, local_filename text not null unique, thumbnail_remote_url text not null default "missing", thumbnail_local_filename text not null default "missing", duration integer not null default 0, view_count integer not null default 0, is_gif boolean default 0, is_downloaded boolean default 0, is_blocked_by_dmca boolean not null default 0, foreign key(chat_message_id) references chat_messages(id) ); create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id); create table chat_message_urls (rowid integer primary key, chat_message_id integer not null, domain text, text text not null, short_text text not null default "", title text, description text, creator_id integer, site_id integer, thumbnail_width integer not null, thumbnail_height integer not null, thumbnail_remote_url text, thumbnail_local_path text, has_card boolean, has_thumbnail boolean, is_content_downloaded boolean default 0, unique (chat_message_id, text) foreign key(chat_message_id) references chat_messages(id) ); create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);`, `create table bookmarks(rowid integer primary key, sort_order integer not null, -- Can't be unique because "-1" is used as "unknown" value user_id integer not null, tweet_id integer not null, unique(user_id, tweet_id) foreign key(tweet_id) references tweets(id) foreign key(user_id) references users(id) ); create index if not exists index_bookmarks_user_id on bookmarks (user_id); create index if not exists index_bookmarks_tweet_id on bookmarks (tweet_id);`, `create table notification_types (rowid integer primary key, name text not null unique ); insert into notification_types(rowid, name) values (1, 'like'), (2, 'retweet'), (3, 'quote-tweet'), (4, 'reply'), (5, 'follow'), (6, 'mention'), (7, 'user is LIVE'), (8, 'poll ended'), (9, 'login'), (10, 'community pinned post'), (11, 'new recommended post'); create table notifications (rowid integer primary key, id text unique, type integer not null, sent_at integer not null, sort_index integer not null, user_id integer not null, -- user who received the notification action_user_id integer references users(id), -- user who triggered the notification action_tweet_id integer references tweets(id), -- tweet associated with the notification action_retweet_id integer references retweets(retweet_id), foreign key(type) references notification_types(rowid) foreign key(user_id) references users(id) ); create table notification_tweets (rowid integer primary key, notification_id not null references notifications(id), tweet_id not null references tweets(id), unique(notification_id, tweet_id) ); create table notification_retweets (rowid integer primary key, notification_id not null references notifications(id), retweet_id not null references retweets(retweet_id), unique(notification_id, retweet_id) ); create table notification_users (rowid integer primary key, notification_id not null references notifications(id), user_id not null references users(id), unique(notification_id, user_id) );`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) // This should only get called on a newly created Profile. // Subsequent updates should change the number, not insert a new row. func InitializeDatabaseVersion(db *sql.DB) { db.MustExec("insert into database_version (version_number) values (?)", ENGINE_DATABASE_VERSION) } func (p Profile) GetDatabaseVersion() (int, error) { row := p.DB.QueryRow("select version_number from database_version") var version int err := row.Scan(&version) if err != nil { return 0, fmt.Errorf("Error checking database version:\n %w", err) } return version, nil } func (p Profile) check_and_update_version() error { version, err := p.GetDatabaseVersion() if err != nil { return err } if version > ENGINE_DATABASE_VERSION { return VersionMismatchError{ENGINE_DATABASE_VERSION, version} } if ENGINE_DATABASE_VERSION > version { fmt.Printf(terminal_utils.COLOR_YELLOW) fmt.Printf("================================================\n") fmt.Printf("Database version is out of date. Upgrading database from version %d to version %d!\n", version, ENGINE_DATABASE_VERSION) fmt.Printf(terminal_utils.COLOR_RESET) return p.UpgradeFromXToY(version, ENGINE_DATABASE_VERSION) } return nil } // Run all the migrations from version X to version Y, and update the `database_version` table's `version_number` func (p Profile) UpgradeFromXToY(x int, y int) error { for i := x; i < y; i++ { fmt.Printf(terminal_utils.COLOR_CYAN) fmt.Println(MIGRATIONS[i]) fmt.Printf(terminal_utils.COLOR_RESET) p.DB.MustExec(MIGRATIONS[i]) p.DB.MustExec("update database_version set version_number = ?", i+1) fmt.Printf(terminal_utils.COLOR_YELLOW) fmt.Printf("Now at database schema version %d.\n", i+1) fmt.Printf(terminal_utils.COLOR_RESET) } fmt.Printf(terminal_utils.COLOR_GREEN) fmt.Printf("================================================\n") fmt.Printf("Database version has been upgraded to version %d.\n", y) fmt.Printf(terminal_utils.COLOR_RESET) return nil }