me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add List queries

Browse Source
This commit is contained in:
Alessio 2024-02-24 16:45:08 -08:00
parent 760b27162e
commit 5b6894eee2
5 changed files with 380 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
pkg/persistence/list_queries.go Normal file
View File

@ -0,0 +1,101 @@
package persistence
import (
"database/sql"
"errors"
"fmt"
. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
)
// Create an empty list, or rename an existing list
func (p Profile) SaveList(l *List) {
// Since the unique column is managed by the database (auto-increment) due to the existence of
// offline lists, we have to check for its existence first
var rowid ListID
if l.IsOnline {
// Online list; look up its rowid by its online ID
// TODO: maybe extract to a function
err := p.DB.Get(&rowid, "select rowid from lists where is_online = 1 and online_list_id = ?", l.ID)
if errors.Is(err, sql.ErrNoRows) {
// Doesn't exist yet
rowid = ListID(0)
} else if err != nil {
panic(err)
}
} else {
// For offline lists, just use the rowid
rowid = l.ID
}
// If `rowid` is 0, then it doesn't exist yet; create it. Otherwise, update it
if rowid == ListID(0) {
result, err := p.DB.NamedExec(`
insert into lists (is_online, online_list_id, name)
values (:is_online, :online_list_id, :name)
`, l)
if err != nil {
panic(err)
}
id, err := result.LastInsertId()
if err != nil {
panic(err)
}
l.ID = ListID(id)
} else {
// Do update
_, err := p.DB.NamedExec(`
update lists set name = :name where rowid = :rowid
`, l)
if err != nil {
panic(err)
}
}
}
func (p Profile) SaveListUsers(list_id ListID, trove TweetTrove) {
for user_id := range trove.Users {
p.SaveListUser(list_id, user_id)
}
}
func (p Profile) SaveListUser(list_id ListID, user_id UserID) {
_, err := p.DB.Exec(`insert into list_users (list_id, user_id) values (?, ?) on conflict do nothing`, list_id, user_id)
if err != nil {
panic(fmt.Errorf("Error executing AddListUser(%d, %d):\n %w", list_id, user_id, err).Error())
}
}
func (p Profile) DeleteListUser(list_id ListID, user_id UserID) {
_, err := p.DB.Exec(`delete from list_users where list_id = ? and user_id = ?`, list_id, user_id)
if err != nil {
panic(fmt.Errorf("Error executing DeleteListUser(%d, %d):\n %w", list_id, user_id, err).Error())
}
}
func (p Profile) GetListById(list_id ListID) List {
var ret List
err := p.DB.Get(&ret, `select rowid, is_online, online_list_id, name from lists where rowid = ?`, list_id)
if err != nil {
panic(err)
}
return ret
}
func (p Profile) GetListUsers(list_id ListID) []User {
var ret []User
err := p.DB.Select(&ret, `
select `+USERS_ALL_SQL_FIELDS+`
from users
where id in (select user_id from list_users where list_id = ?)
`, list_id)
if err != nil {
panic(err)
}
return ret
}
// XXX
// func (p Profile) GetFollowedUsers() List {
// err =
// }

154
pkg/persistence/list_queries_test.go Normal file
View File

@ -0,0 +1,154 @@
package persistence_test
import (
"testing"
"fmt"
"math/rand"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
)
func TestSaveAndLoadOfflineList(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Create an offline list
l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
require.Equal(l.ID, ListID(0))
profile.SaveList(&l)
require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved
// Check it comes back the same
new_l := profile.GetListById(l.ID)
assert.Equal(l.ID, new_l.ID)
assert.Equal(l.IsOnline, new_l.IsOnline)
assert.Equal(l.Name, new_l.Name)
}
func TestRenameOfflineList(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Create an offline list
l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
profile.SaveList(&l)
require.NotEqual(l.ID, ListID(0))
// Rename it
l.Name = fmt.Sprintf("Untest List %d", rand.Int())
profile.SaveList(&l)
// Rename should be effective
new_l := profile.GetListById(l.ID)
assert.Equal(l.ID, new_l.ID)
assert.Equal(l.IsOnline, new_l.IsOnline)
assert.Equal(l.Name, new_l.Name)
}
func TestSaveAndLoadOnlineList(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Create an online list
l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())}
require.Equal(l.ID, ListID(0))
profile.SaveList(&l)
require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved
// Check it comes back the same
new_l := profile.GetListById(l.ID)
assert.Equal(l.ID, new_l.ID)
assert.Equal(l.IsOnline, new_l.IsOnline)
assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists
assert.Equal(l.Name, new_l.Name)
}
func TestRenameOnlineList(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Create an online list
l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())}
profile.SaveList(&l)
require.NotEqual(l.ID, ListID(0))
// Rename it
l.Name = fmt.Sprintf("Untest List %d", rand.Int())
profile.SaveList(&l)
// Rename should be effective
new_l := profile.GetListById(l.ID)
assert.Equal(l.ID, new_l.ID)
assert.Equal(l.IsOnline, new_l.IsOnline)
assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists
assert.Equal(l.Name, new_l.Name)
}
func TestNoOnlineListWithoutID(t *testing.T) {
require := require.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Creating an online list with no OnlineID should fail
l := List{IsOnline: true, OnlineID: OnlineListID(0), Name: fmt.Sprintf("Test List %d", rand.Int())}
defer func() {
// Assert a panic occurred
r, is_ok := recover().(error)
require.True(is_ok)
require.Error(r)
}()
profile.SaveList(&l)
}
func TestAddAndRemoveUserToList(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
// Create a list
l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
profile.SaveList(&l)
// Check there's no users in it
require.Len(profile.GetListUsers(l.ID), 0)
// Add a user to the list
u := create_dummy_user()
require.NoError(profile.SaveUser(&u))
profile.SaveListUser(l.ID, u.ID)
// Make sure it's in the list
users := profile.GetListUsers(l.ID)
require.Len(users, 1)
assert.Equal(u.Handle, users[0].Handle)
// Addding it again should do nothing
profile.SaveListUser(l.ID, u.ID)
require.Len(profile.GetListUsers(l.ID), 1)
// Remove the user from the list
profile.DeleteListUser(l.ID, u.ID)
// Should be gone
require.Len(profile.GetListUsers(l.ID), 0)
}

149
pkg/persistence/schema.sql
View File

@ -1,5 +1,9 @@
PRAGMA foreign_keys = on;
-- Users
-- -----
create table users (rowid integer primary key,
id integer unique not null check(typeof(id) = 'integer'),
display_name text not null,
@ -25,6 +29,42 @@ create table users (rowid integer primary key,
is_content_downloaded boolean default 0
);
create table fake_user_sequence(latest_fake_id integer not null);
insert into fake_user_sequence values(0x4000000000000000);
create table lists(rowid integer primary key,
is_online boolean not null default 0,
online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists
name text not null,
check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0))
check (rowid != 0)
);
create table list_users(rowid integer primary key,
list_id integer not null,
user_id integer not null,
unique(list_id, user_id)
foreign key(list_id) references lists(rowid)
foreign key(user_id) references users(id)
);
create index if not exists index_list_users_list_id on list_users (list_id);
create index if not exists index_list_users_user_id on list_users (user_id);
insert into lists(rowid, name) values (1, "Offline Follows");
insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;
create table follows(rowid integer primary key,
follower_id integer not null,
followee_id integer not null,
unique(follower_id, followee_id),
foreign key(follower_id) references users(id)
foreign key(followee_id) references users(id)
);
create index if not exists index_follows_followee_id on follows (followee_id);
create index if not exists index_follows_follower_id on follows (follower_id);
-- Tweets
-- ------
create table tombstone_types (rowid integer primary key,
short_name text not null unique,
tombstone_text text not null unique
@ -39,7 +79,6 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values
(7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter'),
(8, 'newer-version-available', 'There’s a new version of this Tweet');
create table tweets (rowid integer primary key,
id integer unique not null check(typeof(id) = 'integer'),
user_id integer not null check(typeof(user_id) = 'integer'),
@ -69,15 +108,9 @@ create index if not exists index_tweets_in_reply_to_id on tweets (in_reply_to_id
create index if not exists index_tweets_user_id on tweets (user_id);
create index if not exists index_tweets_posted_at on tweets (posted_at);
create table retweets(rowid integer primary key,
retweet_id integer not null unique,
tweet_id integer not null,
retweeted_by integer not null,
retweeted_at integer not null,
foreign key(tweet_id) references tweets(id)
foreign key(retweeted_by) references users(id)
);
create index if not exists index_retweets_retweeted_at on retweets (retweeted_at);
-- Tweet content
-- -------------
create table urls (rowid integer primary key,
tweet_id integer not null,
@ -124,33 +157,6 @@ create table polls (rowid integer primary key,
);
create index if not exists index_polls_tweet_id on polls (tweet_id);
create table spaces(rowid integer primary key,
id text unique not null,
created_by_id integer,
short_url text not null,
state text not null,
title text not null,
created_at integer not null,
started_at integer not null,
ended_at integer not null,
updated_at integer not null,
is_available_for_replay boolean not null,
replay_watch_count integer,
live_listeners_count integer,
is_details_fetched boolean not null default 0,
foreign key(created_by_id) references users(id)
);
create table space_participants(rowid integer primary key,
user_id integer not null,
space_id not null,
unique(user_id, space_id)
foreign key(space_id) references spaces(id)
-- No foreign key for users, since they may not be downloaded yet and I don't want to
-- download every user who joins a space
);
create table images (rowid integer primary key,
id integer unique not null check(typeof(id) = 'integer'),
@ -192,6 +198,56 @@ create table hashtags (rowid integer primary key,
foreign key(tweet_id) references tweets(id)
);
-- Retweets
-- --------
create table retweets(rowid integer primary key,
retweet_id integer not null unique,
tweet_id integer not null,
retweeted_by integer not null,
retweeted_at integer not null,
foreign key(tweet_id) references tweets(id)
foreign key(retweeted_by) references users(id)
);
create index if not exists index_retweets_retweeted_at on retweets (retweeted_at);
-- Spaces
-- ------
create table spaces(rowid integer primary key,
id text unique not null,
created_by_id integer,
short_url text not null,
state text not null,
title text not null,
created_at integer not null,
started_at integer not null,
ended_at integer not null,
updated_at integer not null,
is_available_for_replay boolean not null,
replay_watch_count integer,
live_listeners_count integer,
is_details_fetched boolean not null default 0,
foreign key(created_by_id) references users(id)
);
create table space_participants(rowid integer primary key,
user_id integer not null,
space_id not null,
unique(user_id, space_id)
foreign key(space_id) references spaces(id)
-- No foreign key for users, since they may not be downloaded yet and I don't want to
-- download every user who joins a space
);
-- Likes
-- -----
create table likes(rowid integer primary key,
sort_order integer not null, -- Can't be unique because "-1" is used as "unknown" value
user_id integer not null,
@ -204,19 +260,8 @@ create index if not exists index_likes_user_id on likes (user_id);
create index if not exists index_likes_tweet_id on likes (tweet_id);
create table follows(rowid integer primary key,
follower_id integer not null,
followee_id integer not null,
unique(follower_id, followee_id),
foreign key(follower_id) references users(id)
foreign key(followee_id) references users(id)
);
create index if not exists index_follows_followee_id on follows (followee_id);
create index if not exists index_follows_follower_id on follows (follower_id);
create table fake_user_sequence(latest_fake_id integer not null);
insert into fake_user_sequence values(0x4000000000000000);
-- Direct Messages (DMs)
-- ---------------------
create table chat_rooms (rowid integer primary key,
id text unique not null,
@ -269,6 +314,10 @@ create table chat_message_reactions (rowid integer primary key,
foreign key(sender_id) references users(id)
);
-- Meta
-- ----
create table database_version(rowid integer primary key,
version_number integer not null unique
);

6
pkg/scraper/list.go
View File

@ -1,10 +1,12 @@
package scraper
type ListID int
type ListID int64
type OnlineListID int64
type List struct {
ID ListID `db:"rowid"`
Type string `db:"type"`
IsOnline bool `db:"is_online"`
OnlineID OnlineListID `db:"online_list_id"`
Name string `db:"name"`
UserIDs []UserID

20
sample_data/seed_data.sql
View File

@ -55,6 +55,26 @@ INSERT INTO users VALUES
(1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116000,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0),
(27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one. Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522000,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0);
create table lists(rowid integer primary key,
is_online boolean not null default 0,
online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists
name text not null,
check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0))
check (rowid != 0)
);
create table list_users(rowid integer primary key,
list_id integer not null,
user_id integer not null,
unique(list_id, user_id)
foreign key(list_id) references lists(rowid)
foreign key(user_id) references users(id)
);
create index if not exists index_list_users_list_id on list_users (list_id);
create index if not exists index_list_users_user_id on list_users (user_id);
insert into lists(rowid, name) values (1, "Offline Follows");
insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;
create table tombstone_types (rowid integer primary key,
short_name text not null unique,
tombstone_text text not null unique