420 lines
17 KiB
Go
420 lines
17 KiB
Go
package persistence
|
||
|
||
import (
|
||
"fmt"
|
||
|
||
sql "github.com/jmoiron/sqlx"
|
||
|
||
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/terminal_utils"
|
||
)
|
||
|
||
type VersionMismatchError struct {
|
||
EngineVersion int
|
||
DatabaseVersion int
|
||
}
|
||
|
||
func (e VersionMismatchError) Error() string {
|
||
return fmt.Sprintf(
|
||
`This profile was created with database schema version %d, which is newer than this application's database schema version, %d.
|
||
Please upgrade this application to a newer version to use this profile. Or downgrade the profile's schema version, somehow.`,
|
||
e.DatabaseVersion, e.EngineVersion,
|
||
)
|
||
}
|
||
|
||
// The Nth entry is the migration that moves you from version N to version N+1.
|
||
var MIGRATIONS = []string{
|
||
`create table polls (rowid integer primary key,
|
||
id integer unique not null check(typeof(id) = 'integer'),
|
||
tweet_id integer not null,
|
||
num_choices integer not null,
|
||
|
||
choice1 text,
|
||
choice1_votes integer,
|
||
choice2 text,
|
||
choice2_votes integer,
|
||
choice3 text,
|
||
choice3_votes integer,
|
||
choice4 text,
|
||
choice4_votes integer,
|
||
|
||
voting_duration integer not null, -- in seconds
|
||
voting_ends_at integer not null,
|
||
|
||
last_scraped_at integer not null,
|
||
|
||
foreign key(tweet_id) references tweets(id)
|
||
);`,
|
||
`alter table tweets add column is_conversation_scraped boolean default 0;
|
||
alter table tweets add column last_scraped_at integer not null default 0`,
|
||
`update tombstone_types set tombstone_text = 'This Tweet is from a suspended account' where rowid = 2;
|
||
insert into tombstone_types (rowid, short_name, tombstone_text)
|
||
values (5, 'violated', 'This Tweet violated the Twitter Rules'),
|
||
(6, 'no longer exists', 'This Tweet is from an account that no longer exists')`,
|
||
`alter table videos add column thumbnail_remote_url text not null default "missing";
|
||
alter table videos add column thumbnail_local_filename text not null default "missing"`,
|
||
|
||
// 5
|
||
`alter table videos add column duration integer not null default 0;
|
||
alter table videos add column view_count integer not null default 0`,
|
||
`alter table users add column is_banned boolean default 0`,
|
||
`alter table urls add column short_text text not null default ""`,
|
||
`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. '
|
||
|| 'This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`,
|
||
`alter table users add column is_followed boolean default 0`,
|
||
|
||
// 10
|
||
`create table fake_user_sequence(latest_fake_id integer not null);
|
||
insert into fake_user_sequence values(0x4000000000000000);
|
||
alter table users add column is_id_fake boolean default 0;`,
|
||
`delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like
|
||
'https://twitter.com/%/status/' || tweets.quoted_tweet_id || "%")`,
|
||
`create table spaces(rowid integer primary key,
|
||
id text unique not null,
|
||
short_url text not null
|
||
);
|
||
alter table tweets add column space_id text references spaces(id)`,
|
||
`alter table videos add column is_blocked_by_dmca boolean not null default 0`,
|
||
`create index if not exists index_tweets_in_reply_to_id on tweets (in_reply_to_id);
|
||
create index if not exists index_urls_tweet_id on urls (tweet_id);
|
||
create index if not exists index_polls_tweet_id on polls (tweet_id);
|
||
create index if not exists index_images_tweet_id on images (tweet_id);
|
||
create index if not exists index_videos_tweet_id on videos (tweet_id);`,
|
||
|
||
// 15
|
||
`alter table spaces add column created_by_id integer references users(id);
|
||
alter table spaces add column state text not null default "";
|
||
alter table spaces add column title text not null default "";
|
||
alter table spaces add column created_at integer;
|
||
alter table spaces add column started_at integer;
|
||
alter table spaces add column ended_at integer;
|
||
alter table spaces add column updated_at integer;
|
||
alter table spaces add column is_available_for_replay boolean not null default 0;
|
||
alter table spaces add column replay_watch_count integer;
|
||
alter table spaces add column live_listeners_count integer;
|
||
alter table spaces add column is_details_fetched boolean not null default 0;
|
||
create table space_participants(rowid integer primary key,
|
||
user_id integer not null,
|
||
space_id not null,
|
||
foreign key(space_id) references spaces(id)
|
||
);`,
|
||
`create index if not exists index_tweets_user_id on tweets (user_id);`,
|
||
`alter table tweets add column is_expandable bool not null default 0;`,
|
||
`create table space_participants_uniq(rowid integer primary key,
|
||
user_id integer not null,
|
||
space_id not null,
|
||
|
||
unique(user_id, space_id)
|
||
foreign key(space_id) references spaces(id)
|
||
-- No foreign key for users, since they may not be downloaded yet and I don't want to
|
||
-- download every user who joins a space
|
||
);
|
||
|
||
insert or replace into space_participants_uniq(rowid, user_id, space_id) select rowid, user_id, space_id from space_participants;
|
||
|
||
drop table space_participants;
|
||
alter table space_participants_uniq rename to space_participants;
|
||
vacuum;`,
|
||
`create table likes(rowid integer primary key,
|
||
sort_order integer unique not null,
|
||
user_id integer not null,
|
||
tweet_id integer not null,
|
||
unique(user_id, tweet_id)
|
||
foreign key(user_id) references users(id)
|
||
foreign key(tweet_id) references tweets(id)
|
||
);`,
|
||
|
||
// 20
|
||
`create index if not exists index_tweets_posted_at on tweets (posted_at);
|
||
create index if not exists index_retweets_retweeted_at on retweets (retweeted_at)`,
|
||
`update spaces set ended_at = ended_at/1000 where ended_at > strftime("%s")*500;
|
||
update spaces set updated_at = updated_at/1000 where updated_at > strftime("%s")*500;
|
||
update spaces set started_at = started_at/1000 where started_at > strftime("%s")*500;
|
||
update spaces set created_at = created_at/1000 where created_at > strftime("%s")*500;`,
|
||
`alter table users add column is_deleted boolean default 0`,
|
||
`begin transaction;
|
||
alter table likes rename to likes_old;
|
||
|
||
create table likes(rowid integer primary key,
|
||
sort_order integer not null,
|
||
user_id integer not null,
|
||
tweet_id integer not null,
|
||
unique(user_id, tweet_id)
|
||
foreign key(user_id) references users(id)
|
||
foreign key(tweet_id) references tweets(id)
|
||
);
|
||
|
||
create index if not exists index_likes_user_id on likes (user_id);
|
||
create index if not exists index_likes_tweet_id on likes (tweet_id);
|
||
insert into likes select * from likes_old;
|
||
drop table likes_old;
|
||
commit;
|
||
vacuum;`,
|
||
`insert into tombstone_types(rowid, short_name, tombstone_text)
|
||
values (8, 'newer-version-available', 'There’s a new version of this Tweet')`,
|
||
|
||
// 25
|
||
`create table chat_rooms (rowid integer primary key,
|
||
id text unique not null,
|
||
type text not null,
|
||
last_messaged_at integer not null,
|
||
is_nsfw boolean not null,
|
||
|
||
-- Group DM info
|
||
created_at integer not null,
|
||
created_by_user_id integer not null,
|
||
name text not null default '',
|
||
avatar_image_remote_url text not null default '',
|
||
avatar_image_local_path text not null default ''
|
||
);
|
||
|
||
create table chat_room_participants(rowid integer primary key,
|
||
chat_room_id text not null,
|
||
user_id integer not null,
|
||
last_read_event_id integer not null,
|
||
is_chat_settings_valid boolean not null default 0,
|
||
is_notifications_disabled boolean not null,
|
||
is_mention_notifications_disabled boolean not null,
|
||
is_read_only boolean not null,
|
||
is_trusted boolean not null,
|
||
is_muted boolean not null,
|
||
status text not null,
|
||
unique(chat_room_id, user_id)
|
||
);
|
||
|
||
create table chat_messages (rowid integer primary key,
|
||
id integer unique not null check(typeof(id) = 'integer'),
|
||
chat_room_id text not null,
|
||
sender_id integer not null,
|
||
sent_at integer not null,
|
||
request_id text not null,
|
||
in_reply_to_id integer,
|
||
text text not null,
|
||
embedded_tweet_id integer not null default 0,
|
||
foreign key(chat_room_id) references chat_rooms(id)
|
||
foreign key(sender_id) references users(id)
|
||
);
|
||
|
||
create table chat_message_reactions (rowid integer primary key,
|
||
id integer unique not null check(typeof(id) = 'integer'),
|
||
message_id integer not null,
|
||
sender_id integer not null,
|
||
sent_at integer not null,
|
||
emoji text not null,
|
||
foreign key(message_id) references chat_messages(id)
|
||
foreign key(sender_id) references users(id)
|
||
);`,
|
||
`create table follows(rowid integer primary key,
|
||
follower_id integer not null,
|
||
followee_id integer not null,
|
||
unique(follower_id, followee_id),
|
||
foreign key(follower_id) references users(id)
|
||
foreign key(followee_id) references users(id)
|
||
);
|
||
create index if not exists index_follows_followee_id on follows (followee_id);
|
||
create index if not exists index_follows_follower_id on follows (follower_id);`,
|
||
`update tweets set
|
||
posted_at = posted_at * 1000,
|
||
last_scraped_at = last_scraped_at * 1000;
|
||
update users set join_date = join_date * 1000;
|
||
update spaces set
|
||
created_at = created_at * 1000,
|
||
started_at = started_at * 1000,
|
||
ended_at = ended_at * 1000,
|
||
updated_at = updated_at * 1000;
|
||
update retweets set retweeted_at = retweeted_at * 1000;
|
||
update polls set
|
||
voting_ends_at = voting_ends_at * 1000,
|
||
last_scraped_at = last_scraped_at * 1000;
|
||
update chat_rooms set created_at = created_at * 1000;`,
|
||
`create table lists(rowid integer primary key,
|
||
is_online boolean not null default 0,
|
||
online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists
|
||
name text not null,
|
||
check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0))
|
||
check (rowid != 0)
|
||
);
|
||
create table list_users(rowid integer primary key,
|
||
list_id integer not null,
|
||
user_id integer not null,
|
||
unique(list_id, user_id)
|
||
foreign key(list_id) references lists(rowid) on delete cascade
|
||
foreign key(user_id) references users(id)
|
||
);
|
||
create index if not exists index_list_users_list_id on list_users (list_id);
|
||
create index if not exists index_list_users_user_id on list_users (user_id);
|
||
insert into lists(rowid, name) values (1, "Offline Follows");
|
||
insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;`,
|
||
`create table chat_message_images (rowid integer primary key,
|
||
id integer unique not null check(typeof(id) = 'integer'),
|
||
chat_message_id integer not null,
|
||
width integer not null,
|
||
height integer not null,
|
||
remote_url text not null unique,
|
||
local_filename text not null unique,
|
||
is_downloaded boolean default 0,
|
||
|
||
foreign key(chat_message_id) references chat_messages(id)
|
||
);
|
||
create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id);
|
||
|
||
create table chat_message_videos (rowid integer primary key,
|
||
id integer unique not null check(typeof(id) = 'integer'),
|
||
chat_message_id integer not null,
|
||
width integer not null,
|
||
height integer not null,
|
||
remote_url text not null unique,
|
||
local_filename text not null unique,
|
||
thumbnail_remote_url text not null default "missing",
|
||
thumbnail_local_filename text not null default "missing",
|
||
duration integer not null default 0,
|
||
view_count integer not null default 0,
|
||
is_gif boolean default 0,
|
||
is_downloaded boolean default 0,
|
||
is_blocked_by_dmca boolean not null default 0,
|
||
|
||
foreign key(chat_message_id) references chat_messages(id)
|
||
);
|
||
create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id);
|
||
|
||
create table chat_message_urls (rowid integer primary key,
|
||
chat_message_id integer not null,
|
||
domain text,
|
||
text text not null,
|
||
short_text text not null default "",
|
||
title text,
|
||
description text,
|
||
creator_id integer,
|
||
site_id integer,
|
||
thumbnail_width integer not null,
|
||
thumbnail_height integer not null,
|
||
thumbnail_remote_url text,
|
||
thumbnail_local_path text,
|
||
has_card boolean,
|
||
has_thumbnail boolean,
|
||
is_content_downloaded boolean default 0,
|
||
|
||
unique (chat_message_id, text)
|
||
foreign key(chat_message_id) references chat_messages(id)
|
||
);
|
||
create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);`,
|
||
`create table bookmarks(rowid integer primary key,
|
||
sort_order integer not null, -- Can't be unique because "-1" is used as "unknown" value
|
||
user_id integer not null,
|
||
tweet_id integer not null,
|
||
unique(user_id, tweet_id)
|
||
foreign key(tweet_id) references tweets(id)
|
||
foreign key(user_id) references users(id)
|
||
);
|
||
create index if not exists index_bookmarks_user_id on bookmarks (user_id);
|
||
create index if not exists index_bookmarks_tweet_id on bookmarks (tweet_id);`,
|
||
`create table notification_types (rowid integer primary key,
|
||
name text not null unique
|
||
);
|
||
insert into notification_types(rowid, name) values
|
||
(1, 'like'),
|
||
(2, 'retweet'),
|
||
(3, 'quote-tweet'),
|
||
(4, 'reply'),
|
||
(5, 'follow'),
|
||
(6, 'mention'),
|
||
(7, 'user is LIVE'),
|
||
(8, 'poll ended'),
|
||
(9, 'login'),
|
||
(10, 'community pinned post'),
|
||
(11, 'new recommended post');
|
||
create table notifications (rowid integer primary key,
|
||
id text unique,
|
||
type integer not null,
|
||
sent_at integer not null,
|
||
sort_index integer not null,
|
||
user_id integer not null, -- user who received the notification
|
||
|
||
action_user_id integer references users(id), -- user who triggered the notification
|
||
action_tweet_id integer references tweets(id), -- tweet associated with the notification
|
||
action_retweet_id integer references retweets(retweet_id),
|
||
|
||
has_detail boolean not null default 0,
|
||
last_scraped_at not null default 0,
|
||
|
||
foreign key(type) references notification_types(rowid)
|
||
foreign key(user_id) references users(id)
|
||
);
|
||
create table notification_tweets (rowid integer primary key,
|
||
notification_id not null references notifications(id),
|
||
tweet_id not null references tweets(id),
|
||
unique(notification_id, tweet_id)
|
||
);
|
||
create table notification_retweets (rowid integer primary key,
|
||
notification_id not null references notifications(id),
|
||
retweet_id not null references retweets(retweet_id),
|
||
unique(notification_id, retweet_id)
|
||
);
|
||
create table notification_users (rowid integer primary key,
|
||
notification_id not null references notifications(id),
|
||
user_id not null references users(id),
|
||
unique(notification_id, user_id)
|
||
);`,
|
||
}
|
||
var ENGINE_DATABASE_VERSION = len(MIGRATIONS)
|
||
|
||
// This should only get called on a newly created Profile.
|
||
// Subsequent updates should change the number, not insert a new row.
|
||
func InitializeDatabaseVersion(db *sql.DB) {
|
||
db.MustExec("insert into database_version (version_number) values (?)", ENGINE_DATABASE_VERSION)
|
||
}
|
||
|
||
func (p Profile) GetDatabaseVersion() (int, error) {
|
||
row := p.DB.QueryRow("select version_number from database_version")
|
||
|
||
var version int
|
||
|
||
err := row.Scan(&version)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("Error checking database version:\n %w", err)
|
||
}
|
||
return version, nil
|
||
}
|
||
|
||
func (p Profile) check_and_update_version() error {
|
||
version, err := p.GetDatabaseVersion()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
if version > ENGINE_DATABASE_VERSION {
|
||
return VersionMismatchError{ENGINE_DATABASE_VERSION, version}
|
||
}
|
||
|
||
if ENGINE_DATABASE_VERSION > version {
|
||
fmt.Printf(terminal_utils.COLOR_YELLOW)
|
||
fmt.Printf("================================================\n")
|
||
fmt.Printf("Database version is out of date. Upgrading database from version %d to version %d!\n", version,
|
||
ENGINE_DATABASE_VERSION)
|
||
fmt.Printf(terminal_utils.COLOR_RESET)
|
||
return p.UpgradeFromXToY(version, ENGINE_DATABASE_VERSION)
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// Run all the migrations from version X to version Y, and update the `database_version` table's `version_number`
|
||
func (p Profile) UpgradeFromXToY(x int, y int) error {
|
||
for i := x; i < y; i++ {
|
||
fmt.Printf(terminal_utils.COLOR_CYAN)
|
||
fmt.Println(MIGRATIONS[i])
|
||
fmt.Printf(terminal_utils.COLOR_RESET)
|
||
|
||
p.DB.MustExec(MIGRATIONS[i])
|
||
p.DB.MustExec("update database_version set version_number = ?", i+1)
|
||
|
||
fmt.Printf(terminal_utils.COLOR_YELLOW)
|
||
fmt.Printf("Now at database schema version %d.\n", i+1)
|
||
fmt.Printf(terminal_utils.COLOR_RESET)
|
||
}
|
||
fmt.Printf(terminal_utils.COLOR_GREEN)
|
||
fmt.Printf("================================================\n")
|
||
fmt.Printf("Database version has been upgraded to version %d.\n", y)
|
||
fmt.Printf(terminal_utils.COLOR_RESET)
|
||
return nil
|
||
}
|