From b77612c66f61f034777a360b0fa5bc578cd3247b Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 25 Aug 2024 22:54:18 -0700 Subject: [PATCH] Add SQL queries for notifications --- pkg/persistence/notification_queries.go | 70 ++++++++++++++++++++ pkg/persistence/notification_queries_test.go | 22 ++++++ pkg/persistence/schema.sql | 45 +++++++++++++ pkg/persistence/tweet_trove_queries.go | 4 ++ pkg/persistence/utils_test.go | 17 +++++ pkg/persistence/versions.go | 39 +++++++++++ pkg/scraper/notification.go | 16 ++--- 7 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 pkg/persistence/notification_queries.go create mode 100644 pkg/persistence/notification_queries_test.go diff --git a/pkg/persistence/notification_queries.go b/pkg/persistence/notification_queries.go new file mode 100644 index 0000000..ea5a924 --- /dev/null +++ b/pkg/persistence/notification_queries.go @@ -0,0 +1,70 @@ +package persistence + +import ( + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +func (p Profile) SaveNotification(n Notification) { + tx, err := p.DB.Beginx() + if err != nil { + panic(err) + } + + // Save the Notification + _, err = tx.NamedExec(` + insert into notifications(id, type, sent_at, sort_index, user_id, action_user_id, action_tweet_id, action_retweet_id) + values (:id, :type, :sent_at, :sort_index, :user_id, nullif(:action_user_id, 0), nullif(:action_tweet_id, 0), + nullif(:action_retweet_id, 0)) + on conflict do update + set sent_at = max(sent_at, :sent_at), + sort_index = max(sort_index, :sort_index), + action_user_id = :action_user_id, + action_tweet_id = :action_tweet_id + `, n) + if err != nil { + panic(err) + } + + // Save relevant users and tweets + for _, u_id := range n.UserIDs { + _, err = tx.Exec(` + insert into notification_users(notification_id, user_id) values (?, ?) on conflict do nothing + `, n.ID, u_id) + if err != nil { + panic(err) + } + } + for _, t_id := range n.TweetIDs { + _, err = tx.Exec(` + insert into notification_tweets(notification_id, tweet_id) values (?, ?) on conflict do nothing + `, n.ID, t_id) + if err != nil { + panic(err) + } + } + err = tx.Commit() + if err != nil { + panic(err) + } +} + +func (p Profile) GetNotification(id NotificationID) Notification { + var ret Notification + err := p.DB.Get(&ret, + `select id, type, sent_at, sort_index, user_id, ifnull(action_user_id, 0) action_user_id, + ifnull(action_tweet_id, 0) action_tweet_id, ifnull(action_retweet_id, 0) action_retweet_id + from notifications where id = ?`, + id) + if err != nil { + panic(err) + } + err = p.DB.Select(&ret.UserIDs, `select user_id from notification_users where notification_id = ?`, id) + if err != nil { + panic(err) + } + err = p.DB.Select(&ret.TweetIDs, `select tweet_id from notification_tweets where notification_id = ?`, id) + if err != nil { + panic(err) + } + return ret +} diff --git a/pkg/persistence/notification_queries_test.go b/pkg/persistence/notification_queries_test.go new file mode 100644 index 0000000..ffe2af2 --- /dev/null +++ b/pkg/persistence/notification_queries_test.go @@ -0,0 +1,22 @@ +package persistence_test + +import ( + "testing" + + "github.com/go-test/deep" +) + +func TestSaveAndLoadNotification(t *testing.T) { + profile_path := "test_profiles/TestNotificationQuery" + profile := create_or_load_profile(profile_path) + + // Save it + n := create_dummy_notification() + profile.SaveNotification(n) + + // Check it comes back the same + new_n := profile.GetNotification(n.ID) + if diff := deep.Equal(n, new_n); diff != nil { + t.Error(diff) + } +} diff --git a/pkg/persistence/schema.sql b/pkg/persistence/schema.sql index 4ab9cfe..d96ce2c 100644 --- a/pkg/persistence/schema.sql +++ b/pkg/persistence/schema.sql @@ -384,6 +384,51 @@ create table chat_message_urls (rowid integer primary key, create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id); +-- Notifications +-- ------------- + +create table notification_types (rowid integer primary key, + name text not null unique +); +insert into notification_types(rowid, name) values + (1, 'like'), + (2, 'retweet'), + (3, 'quote-tweet'), + (4, 'reply'), + (5, 'follow'), + (6, 'mention'), + (7, 'user is LIVE'), + (8, 'poll ended'), + (9, 'login'), + (10, 'community pinned post'), + (11, 'new recommended post'); + +create table notifications (rowid integer primary key, + id text unique, + type integer not null, + sent_at integer not null, + sort_index integer not null, + user_id integer not null, -- user who received the notification + + action_user_id integer references users(id), -- user who triggered the notification + action_tweet_id integer references tweets(id), -- tweet associated with the notification + action_retweet_id integer references retweets(retweet_id), + + foreign key(type) references notification_types(rowid) + foreign key(user_id) references users(id) +); +create table notification_tweets (rowid integer primary key, + notification_id not null references notifications(id), + tweet_id not null references tweets(id), + unique(notification_id, tweet_id) +); +create table notification_users (rowid integer primary key, + notification_id not null references notifications(id), + user_id not null references users(id), + unique(notification_id, user_id) +); + + -- Meta -- ---- diff --git a/pkg/persistence/tweet_trove_queries.go b/pkg/persistence/tweet_trove_queries.go index 0d1bdbc..d0a55c8 100644 --- a/pkg/persistence/tweet_trove_queries.go +++ b/pkg/persistence/tweet_trove_queries.go @@ -96,6 +96,10 @@ func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool, api *API } } + for _, n := range trove.Notifications { + p.SaveNotification(n) + } + // DM related content // ------------------ diff --git a/pkg/persistence/utils_test.go b/pkg/persistence/utils_test.go index d2730e1..1811ebd 100644 --- a/pkg/persistence/utils_test.go +++ b/pkg/persistence/utils_test.go @@ -396,3 +396,20 @@ func create_dummy_chat_message() DMMessage { Urls: []Url{url}, } } + +func create_dummy_notification() Notification { + rand.Seed(time.Now().UnixNano()) + id := NotificationID(fmt.Sprintf("Notification #%d", rand.Int())) + + return Notification{ + ID: id, + Type: NOTIFICATION_TYPE_REPLY, + SentAt: TimestampFromUnix(10000), + SortIndex: rand.Int63(), + UserID: create_stable_user().ID, + ActionUserID: create_stable_user().ID, + ActionTweetID: create_stable_tweet().ID, + TweetIDs: []TweetID{create_stable_tweet().ID}, + UserIDs: []UserID{create_stable_user().ID}, + } +} diff --git a/pkg/persistence/versions.go b/pkg/persistence/versions.go index 5ce6ca7..9ad7aad 100644 --- a/pkg/persistence/versions.go +++ b/pkg/persistence/versions.go @@ -307,6 +307,45 @@ var MIGRATIONS = []string{ ); create index if not exists index_bookmarks_user_id on bookmarks (user_id); create index if not exists index_bookmarks_tweet_id on bookmarks (tweet_id);`, + `create table notification_types (rowid integer primary key, + name text not null unique + ); + insert into notification_types(rowid, name) values + (1, 'like'), + (2, 'retweet'), + (3, 'quote-tweet'), + (4, 'reply'), + (5, 'follow'), + (6, 'mention'), + (7, 'user is LIVE'), + (8, 'poll ended'), + (9, 'login'), + (10, 'community pinned post'), + (11, 'new recommended post'); + create table notifications (rowid integer primary key, + id text unique, + type integer not null, + sent_at integer not null, + sort_index integer not null, + user_id integer not null, -- user who received the notification + + action_user_id integer references users(id), -- user who triggered the notification + action_tweet_id integer references tweets(id), -- tweet associated with the notification + action_retweet_id integer references retweets(retweet_id), + + foreign key(type) references notification_types(rowid) + foreign key(user_id) references users(id) + ); + create table notification_tweets (rowid integer primary key, + notification_id not null references notifications(id), + tweet_id not null references tweets(id), + unique(notification_id, tweet_id) + ); + create table notification_users (rowid integer primary key, + notification_id not null references notifications(id), + user_id not null references users(id), + unique(notification_id, user_id) + );`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) diff --git a/pkg/scraper/notification.go b/pkg/scraper/notification.go index 8c67bd1..3ec8f41 100644 --- a/pkg/scraper/notification.go +++ b/pkg/scraper/notification.go @@ -19,15 +19,15 @@ const ( ) type Notification struct { - ID NotificationID - Type int - SentAt Timestamp - SortIndex int64 - UserID UserID // recipient of the notification + ID NotificationID `db:"id"` + Type int `db:"type"` + SentAt Timestamp `db:"sent_at"` + SortIndex int64 `db:"sort_index"` + UserID UserID `db:"user_id"` // recipient of the notification - ActionUserID UserID - ActionTweetID TweetID - ActionRetweetID TweetID + ActionUserID UserID `db:"action_user_id"` + ActionTweetID TweetID `db:"action_tweet_id"` + ActionRetweetID TweetID `db:"action_retweet_id"` TweetIDs []TweetID UserIDs []UserID