Add List queries

pkg/persistence/list_queries.go

@@ -0,0 +1,101 @@
+package persistence
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+
+	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+)
+
+// Create an empty list, or rename an existing list
+func (p Profile) SaveList(l *List) {
+	// Since the unique column is managed by the database (auto-increment) due to the existence of
+	// offline lists, we have to check for its existence first
+	var rowid ListID
+	if l.IsOnline {
+		// Online list; look up its rowid by its online ID
+		// TODO: maybe extract to a function
+		err := p.DB.Get(&rowid, "select rowid from lists where is_online = 1 and online_list_id = ?", l.ID)
+		if errors.Is(err, sql.ErrNoRows) {
+			// Doesn't exist yet
+			rowid = ListID(0)
+		} else if err != nil {
+			panic(err)
+		}
+	} else {
+		// For offline lists, just use the rowid
+		rowid = l.ID
+	}
+
+	// If `rowid` is 0, then it doesn't exist yet; create it.  Otherwise, update it
+	if rowid == ListID(0) {
+		result, err := p.DB.NamedExec(`
+		    insert into lists (is_online, online_list_id, name)
+		    values (:is_online, :online_list_id, :name)
+		`, l)
+		if err != nil {
+			panic(err)
+		}
+		id, err := result.LastInsertId()
+		if err != nil {
+			panic(err)
+		}
+		l.ID = ListID(id)
+	} else {
+		// Do update
+		_, err := p.DB.NamedExec(`
+			update lists set name = :name where rowid = :rowid
+		`, l)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func (p Profile) SaveListUsers(list_id ListID, trove TweetTrove) {
+	for user_id := range trove.Users {
+		p.SaveListUser(list_id, user_id)
+	}
+}
+
+func (p Profile) SaveListUser(list_id ListID, user_id UserID) {
+	_, err := p.DB.Exec(`insert into list_users (list_id, user_id) values (?, ?) on conflict do nothing`, list_id, user_id)
+	if err != nil {
+		panic(fmt.Errorf("Error executing AddListUser(%d, %d):\n  %w", list_id, user_id, err).Error())
+	}
+}
+
+func (p Profile) DeleteListUser(list_id ListID, user_id UserID) {
+	_, err := p.DB.Exec(`delete from list_users where list_id = ? and user_id = ?`, list_id, user_id)
+	if err != nil {
+		panic(fmt.Errorf("Error executing DeleteListUser(%d, %d):\n  %w", list_id, user_id, err).Error())
+	}
+}
+
+func (p Profile) GetListById(list_id ListID) List {
+	var ret List
+	err := p.DB.Get(&ret, `select rowid, is_online, online_list_id, name from lists where rowid = ?`, list_id)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
+func (p Profile) GetListUsers(list_id ListID) []User {
+	var ret []User
+	err := p.DB.Select(&ret, `
+	    select `+USERS_ALL_SQL_FIELDS+`
+	      from users
+	     where id in (select user_id from list_users where list_id = ?)
+	`, list_id)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
+// XXX
+// func (p Profile) GetFollowedUsers() List {
+// 	err =
+// }

pkg/persistence/list_queries_test.go

@@ -0,0 +1,154 @@
+package persistence_test
+
+import (
+	"testing"
+
+	"fmt"
+	"math/rand"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
+	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+)
+
+func TestSaveAndLoadOfflineList(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Create an offline list
+	l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
+	require.Equal(l.ID, ListID(0))
+	profile.SaveList(&l)
+	require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved
+
+	// Check it comes back the same
+	new_l := profile.GetListById(l.ID)
+	assert.Equal(l.ID, new_l.ID)
+	assert.Equal(l.IsOnline, new_l.IsOnline)
+	assert.Equal(l.Name, new_l.Name)
+}
+
+func TestRenameOfflineList(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Create an offline list
+	l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
+	profile.SaveList(&l)
+	require.NotEqual(l.ID, ListID(0))
+
+	// Rename it
+	l.Name = fmt.Sprintf("Untest List %d", rand.Int())
+	profile.SaveList(&l)
+
+	// Rename should be effective
+	new_l := profile.GetListById(l.ID)
+	assert.Equal(l.ID, new_l.ID)
+	assert.Equal(l.IsOnline, new_l.IsOnline)
+	assert.Equal(l.Name, new_l.Name)
+}
+
+func TestSaveAndLoadOnlineList(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Create an online list
+	l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())}
+	require.Equal(l.ID, ListID(0))
+	profile.SaveList(&l)
+	require.NotEqual(l.ID, ListID(0)) // ID should be assigned when it's saved
+
+	// Check it comes back the same
+	new_l := profile.GetListById(l.ID)
+	assert.Equal(l.ID, new_l.ID)
+	assert.Equal(l.IsOnline, new_l.IsOnline)
+	assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists
+	assert.Equal(l.Name, new_l.Name)
+}
+
+func TestRenameOnlineList(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Create an online list
+	l := List{IsOnline: true, OnlineID: OnlineListID(rand.Int()), Name: fmt.Sprintf("Test List %d", rand.Int())}
+	profile.SaveList(&l)
+	require.NotEqual(l.ID, ListID(0))
+
+	// Rename it
+	l.Name = fmt.Sprintf("Untest List %d", rand.Int())
+	profile.SaveList(&l)
+
+	// Rename should be effective
+	new_l := profile.GetListById(l.ID)
+	assert.Equal(l.ID, new_l.ID)
+	assert.Equal(l.IsOnline, new_l.IsOnline)
+	assert.Equal(l.OnlineID, new_l.OnlineID) // Check OnlineID for online lists
+	assert.Equal(l.Name, new_l.Name)
+}
+
+func TestNoOnlineListWithoutID(t *testing.T) {
+	require := require.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Creating an online list with no OnlineID should fail
+	l := List{IsOnline: true, OnlineID: OnlineListID(0), Name: fmt.Sprintf("Test List %d", rand.Int())}
+	defer func() {
+		// Assert a panic occurred
+		r, is_ok := recover().(error)
+		require.True(is_ok)
+		require.Error(r)
+	}()
+	profile.SaveList(&l)
+}
+
+func TestAddAndRemoveUserToList(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	// Create a list
+	l := List{IsOnline: false, Name: fmt.Sprintf("Test List %d", rand.Int())}
+	profile.SaveList(&l)
+
+	// Check there's no users in it
+	require.Len(profile.GetListUsers(l.ID), 0)
+
+	// Add a user to the list
+	u := create_dummy_user()
+	require.NoError(profile.SaveUser(&u))
+	profile.SaveListUser(l.ID, u.ID)
+
+	// Make sure it's in the list
+	users := profile.GetListUsers(l.ID)
+	require.Len(users, 1)
+	assert.Equal(u.Handle, users[0].Handle)
+
+	// Addding it again should do nothing
+	profile.SaveListUser(l.ID, u.ID)
+	require.Len(profile.GetListUsers(l.ID), 1)
+
+	// Remove the user from the list
+	profile.DeleteListUser(l.ID, u.ID)
+
+	// Should be gone
+	require.Len(profile.GetListUsers(l.ID), 0)
+}

pkg/persistence/schema.sql

@@ -1,5 +1,9 @@
 PRAGMA foreign_keys = on;
 
+
+-- Users
+-- -----
+
 create table users (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
     display_name text not null,
@@ -25,6 +29,42 @@ create table users (rowid integer primary key,
     is_content_downloaded boolean default 0
 );
 
+create table fake_user_sequence(latest_fake_id integer not null);
+insert into fake_user_sequence values(0x4000000000000000);
+
+create table lists(rowid integer primary key,
+    is_online boolean not null default 0,
+    online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists
+    name text not null,
+    check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0))
+    check (rowid != 0)
+);
+create table list_users(rowid integer primary key,
+    list_id integer not null,
+    user_id integer not null,
+    unique(list_id, user_id)
+    foreign key(list_id) references lists(rowid)
+    foreign key(user_id) references users(id)
+);
+create index if not exists index_list_users_list_id on list_users (list_id);
+create index if not exists index_list_users_user_id on list_users (user_id);
+insert into lists(rowid, name) values (1, "Offline Follows");
+insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;
+
+create table follows(rowid integer primary key,
+    follower_id integer not null,
+    followee_id integer not null,
+    unique(follower_id, followee_id),
+    foreign key(follower_id) references users(id)
+    foreign key(followee_id) references users(id)
+);
+create index if not exists index_follows_followee_id on follows (followee_id);
+create index if not exists index_follows_follower_id on follows (follower_id);
+
+
+-- Tweets
+-- ------
+
 create table tombstone_types (rowid integer primary key,
     short_name text not null unique,
     tombstone_text text not null unique
@@ -39,7 +79,6 @@ insert into tombstone_types(rowid, short_name, tombstone_text) values
     (7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter'),
     (8, 'newer-version-available', 'There’s a new version of this Tweet');
 
-
 create table tweets (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
     user_id integer not null check(typeof(user_id) = 'integer'),
@@ -69,15 +108,9 @@ create index if not exists index_tweets_in_reply_to_id on tweets (in_reply_to_id
 create index if not exists index_tweets_user_id        on tweets (user_id);
 create index if not exists index_tweets_posted_at      on tweets (posted_at);
 
-create table retweets(rowid integer primary key,
+
-    retweet_id integer not null unique,
+-- Tweet content
-    tweet_id integer not null,
+-- -------------
-    retweeted_by integer not null,
-    retweeted_at integer not null,
-    foreign key(tweet_id) references tweets(id)
-    foreign key(retweeted_by) references users(id)
-);
-create index if not exists index_retweets_retweeted_at on retweets (retweeted_at);
 
 create table urls (rowid integer primary key,
     tweet_id integer not null,
@@ -124,33 +157,6 @@ create table polls (rowid integer primary key,
 );
 create index if not exists index_polls_tweet_id on polls (tweet_id);
 
-create table spaces(rowid integer primary key,
-    id text unique not null,
-    created_by_id integer,
-    short_url text not null,
-    state text not null,
-    title text not null,
-    created_at integer not null,
-    started_at integer not null,
-    ended_at integer not null,
-    updated_at integer not null,
-    is_available_for_replay boolean not null,
-    replay_watch_count integer,
-    live_listeners_count integer,
-    is_details_fetched boolean not null default 0,
-
-    foreign key(created_by_id) references users(id)
-);
-
-create table space_participants(rowid integer primary key,
-    user_id integer not null,
-    space_id not null,
-
-    unique(user_id, space_id)
-    foreign key(space_id) references spaces(id)
-    -- No foreign key for users, since they may not be downloaded yet and I don't want to
-    -- download every user who joins a space
-);
 
 create table images (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
@@ -192,6 +198,56 @@ create table hashtags (rowid integer primary key,
     foreign key(tweet_id) references tweets(id)
 );
 
+
+-- Retweets
+-- --------
+
+create table retweets(rowid integer primary key,
+    retweet_id integer not null unique,
+    tweet_id integer not null,
+    retweeted_by integer not null,
+    retweeted_at integer not null,
+    foreign key(tweet_id) references tweets(id)
+    foreign key(retweeted_by) references users(id)
+);
+create index if not exists index_retweets_retweeted_at on retweets (retweeted_at);
+
+
+-- Spaces
+-- ------
+
+create table spaces(rowid integer primary key,
+    id text unique not null,
+    created_by_id integer,
+    short_url text not null,
+    state text not null,
+    title text not null,
+    created_at integer not null,
+    started_at integer not null,
+    ended_at integer not null,
+    updated_at integer not null,
+    is_available_for_replay boolean not null,
+    replay_watch_count integer,
+    live_listeners_count integer,
+    is_details_fetched boolean not null default 0,
+
+    foreign key(created_by_id) references users(id)
+);
+
+create table space_participants(rowid integer primary key,
+    user_id integer not null,
+    space_id not null,
+
+    unique(user_id, space_id)
+    foreign key(space_id) references spaces(id)
+    -- No foreign key for users, since they may not be downloaded yet and I don't want to
+    -- download every user who joins a space
+);
+
+
+-- Likes
+-- -----
+
 create table likes(rowid integer primary key,
     sort_order integer not null, -- Can't be unique because "-1" is used as "unknown" value
     user_id integer not null,
@@ -204,19 +260,8 @@ create index if not exists index_likes_user_id on likes (user_id);
 create index if not exists index_likes_tweet_id on likes (tweet_id);
 
 
-create table follows(rowid integer primary key,
+-- Direct Messages (DMs)
-    follower_id integer not null,
+-- ---------------------
-    followee_id integer not null,
-    unique(follower_id, followee_id),
-    foreign key(follower_id) references users(id)
-    foreign key(followee_id) references users(id)
-);
-create index if not exists index_follows_followee_id on follows (followee_id);
-create index if not exists index_follows_follower_id on follows (follower_id);
-
-
-create table fake_user_sequence(latest_fake_id integer not null);
-insert into fake_user_sequence values(0x4000000000000000);
 
 create table chat_rooms (rowid integer primary key,
     id text unique not null,
@@ -269,6 +314,10 @@ create table chat_message_reactions (rowid integer primary key,
     foreign key(sender_id) references users(id)
 );
 
+
+-- Meta
+-- ----
+
 create table database_version(rowid integer primary key,
     version_number integer not null unique
 );

pkg/scraper/list.go

@@ -1,11 +1,13 @@
 package scraper
 
-type ListID int
+type ListID int64
+type OnlineListID int64
 
 type List struct {
-	ID   ListID `db:"rowid"`
+	ID       ListID       `db:"rowid"`
-	Type string `db:"type"`
+	IsOnline bool         `db:"is_online"`
-	Name string `db:"name"`
+	OnlineID OnlineListID `db:"online_list_id"`
+	Name     string       `db:"name"`
 
 	UserIDs []UserID
 	Users   []*User

sample_data/seed_data.sql

@@ -55,6 +55,26 @@ INSERT INTO users VALUES
     (1680,1458284524761075714,'wispem-wantex','wispem_wantex',replace('~wispem-wantex\n\nCurrently looking for work (DMs open)','\n',char(10)),136,483,'on my computer','https://offline-twitter.com/',1636517116000,0,0,0,'https://pbs.twimg.com/profile_images/1462880679687954433/dXJN4Bo4.jpg','wispem_wantex_profile_dXJN4Bo4.jpg','','',1695221528617468324,1,0,0,0),
     (27398,1488963321701171204,'Offline Twatter','Offline_Twatter',replace('Offline Twitter is an open source twitter client and tweet-archiving app all in one.  Try it out!\n\nSource code: https://t.co/2PMumKSxFO','\n',char(10)),4,2,'','https://offline-twitter.com',1643831522000,0,0,0,'https://pbs.twimg.com/profile_images/1507883049853210626/TytFbk_3.jpg','Offline_Twatter_profile_TytFbk_3.jpg','','',1507883724615999488,1,1,0,0);
 
+create table lists(rowid integer primary key,
+    is_online boolean not null default 0,
+    online_list_id integer not null default 0, -- Will be 0 for lists that aren't Twitter Lists
+    name text not null,
+    check ((is_online = 0 and online_list_id = 0) or (is_online != 0 and online_list_id != 0))
+    check (rowid != 0)
+);
+create table list_users(rowid integer primary key,
+    list_id integer not null,
+    user_id integer not null,
+    unique(list_id, user_id)
+    foreign key(list_id) references lists(rowid)
+    foreign key(user_id) references users(id)
+);
+create index if not exists index_list_users_list_id on list_users (list_id);
+create index if not exists index_list_users_user_id on list_users (user_id);
+insert into lists(rowid, name) values (1, "Offline Follows");
+insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;
+
+
 create table tombstone_types (rowid integer primary key,
     short_name text not null unique,
     tombstone_text text not null unique