me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track retweet lists in notifications alongside tweets

Browse Source
This commit is contained in:
Alessio 2024-08-28 19:20:29 -07:00
parent 0a1cdc17af
commit dc816c6f28
8 changed files with 135 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/persistence/notification_queries.go
View File

@ -39,6 +39,16 @@ func (p Profile) SaveNotification(n Notification) {
insert into notification_tweets(notification_id, tweet_id) values (?, ?) on conflict do nothing
`, n.ID, t_id)
if err != nil {
fmt.Printf("failed to save notification %#v\n", n)
panic(err)
}
}
for _, r_id := range n.RetweetIDs {
_, err = tx.Exec(`
insert into notification_retweets(notification_id, retweet_id) values (?, ?) on conflict do nothing
`, n.ID, r_id)
if err != nil {
fmt.Printf("failed to save notification %#v\n", n)
panic(err)
}
}
@ -66,5 +76,9 @@ func (p Profile) GetNotification(id NotificationID) Notification {
if err != nil {
panic(err)
}
err = p.DB.Select(&ret.RetweetIDs, `select retweet_id from notification_retweets where notification_id = ?`, id)
if err != nil {
panic(err)
}
return ret
}

5
pkg/persistence/schema.sql
View File

@ -422,6 +422,11 @@ create table notification_tweets (rowid integer primary key,
tweet_id not null references tweets(id),
unique(notification_id, tweet_id)
);
create table notification_retweets (rowid integer primary key,
notification_id not null references notifications(id),
retweet_id not null references retweets(retweet_id),
unique(notification_id, retweet_id)
);
create table notification_users (rowid integer primary key,
notification_id not null references notifications(id),
user_id not null references users(id),

2
pkg/persistence/utils_test.go
View File

@ -409,7 +409,9 @@ func create_dummy_notification() Notification {
UserID: create_stable_user().ID,
ActionUserID: create_stable_user().ID,
ActionTweetID: create_stable_tweet().ID,
ActionRetweetID: create_stable_retweet().RetweetID,
TweetIDs: []TweetID{create_stable_tweet().ID},
UserIDs: []UserID{create_stable_user().ID},
RetweetIDs: []TweetID{create_stable_retweet().RetweetID},
}
}

5
pkg/persistence/versions.go
View File

@ -341,6 +341,11 @@ var MIGRATIONS = []string{
tweet_id not null references tweets(id),
unique(notification_id, tweet_id)
);
create table notification_retweets (rowid integer primary key,
notification_id not null references notifications(id),
retweet_id not null references retweets(retweet_id),
unique(notification_id, retweet_id)
);
create table notification_users (rowid integer primary key,
notification_id not null references notifications(id),
user_id not null references users(id),

48
pkg/scraper/api_types_notifications.go
View File

@ -92,28 +92,29 @@ func (t *TweetResponse) ToTweetTroveAsNotifications(current_user_id UserID) (Twe
} else if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "mentioned") {
notification.Type = NOTIFICATION_TYPE_MENTION
}
if entry.Content.Item.Content.Tweet.ID != 0 {
notification.ActionTweetID = TweetID(entry.Content.Item.Content.Tweet.ID)
notification.TweetIDs = []TweetID{notification.ActionTweetID}
notification.ActionUserID = UserID(ret.Tweets[notification.ActionTweetID].UserID)
}
if entry.Content.Item.Content.Notification.ID != "" {
notification.UserIDs = []UserID{}
for _, u_id := range entry.Content.Item.Content.Notification.FromUsers {
notification.UserIDs = append(notification.UserIDs, UserID(u_id))
notification.ActionUserID = UserID(u_id)
}
notification.TweetIDs = []TweetID{}
for _, t_id := range entry.Content.Item.Content.Notification.TargetTweets {
notification.TweetIDs = append(notification.TweetIDs, TweetID(t_id))
notification.ActionTweetID = TweetID(t_id)
} else if notification.ActionTweetID != TweetID(0) {
// Check if it's a tweet or retweet
_, is_ok := ret.Retweets[notification.ActionTweetID]
if is_ok {
// It's a retweet, not a tweet
notification.ActionRetweetID = notification.ActionTweetID
notification.RetweetIDs = []TweetID{notification.ActionRetweetID}
notification.ActionTweetID = TweetID(0) // It's not a tweet
} else {
// Otherwise, it's a tweet
notification.TweetIDs = []TweetID{notification.ActionTweetID}
}
}
ret.Notifications[notification.ID] = notification
}
}
return ret, err
return ret, nil
}
func ParseSingleNotification(n APINotification) Notification {
@ -128,8 +129,8 @@ func ParseSingleNotification(n APINotification) Notification {
n.Message.Text = string(runetext[0:from]) + string(runetext[to:])
}
// t.Entities.ReplyMentions = strings.TrimSpace(string([]rune(t.FullText)[0:t.DisplayTextRange[0]]))
// Try to identify the notification type from the notification object itself (not always possible)
if strings.HasSuffix(n.Message.Text, "followed you") {
ret.Type = NOTIFICATION_TYPE_FOLLOW
} else if strings.Contains(n.Message.Text, "liked") {
@ -139,24 +140,23 @@ func ParseSingleNotification(n APINotification) Notification {
} else if strings.Contains(n.Message.Text, "There was a login to your account") {
ret.Type = NOTIFICATION_TYPE_LOGIN
}
// TODO: more types?
ret.SentAt = TimestampFromUnixMilli(n.TimestampMs)
// Process UserIDs
ret.UserIDs = []UserID{}
for _, u := range n.Template.AggregateUserActionsV1.FromUsers {
ret.UserIDs = append(ret.UserIDs, UserID(u.User.ID))
if ret.ActionUserID == UserID(0) {
// "Liking" users are ordered most-recent-first
ret.ActionUserID = UserID(u.User.ID)
}
}
// If the action has a "target", store it temporarily in ActionTweetID
target_objs := n.Template.AggregateUserActionsV1.TargetObjects
if len(target_objs) > 0 {
if strings.HasSuffix(n.Message.Text, "liked your repost") {
// Retweet
ret.ActionRetweetID = TweetID(target_objs[0].Tweet.ID)
} else {
// Normal tweet
ret.ActionTweetID = TweetID(target_objs[0].Tweet.ID)
ret.TweetIDs = []TweetID{TweetID(target_objs[0].Tweet.ID)}
}
ret.ActionTweetID = TweetID(target_objs[0].Tweet.ID)
}
return ret

54
pkg/scraper/api_types_notifications_test.go
View File

@ -29,8 +29,10 @@ func TestParseNotificationsPage(t *testing.T) {
notif1, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BFN3re-ZsU"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_LOGIN, notif1.Type)
assert.Equal(int64(1723851817578), notif1.SortIndex)
assert.Equal(current_user_id, notif1.UserID)
// Simple retweet: 1 user retweets 1 tweet
notif2, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BFaOkNV8aw"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_RETWEET, notif2.Type)
@ -38,13 +40,24 @@ func TestParseNotificationsPage(t *testing.T) {
assert.Equal(UserID(1458284524761075714), notif2.ActionUserID)
assert.Equal(TweetID(1824915465275392037), notif2.ActionTweetID)
assert.Equal(TimestampFromUnixMilli(1723928739342), notif2.SentAt)
assert.Len(notif2.UserIDs, 1)
assert.Contains(notif2.UserIDs, UserID(1458284524761075714))
assert.Len(notif2.TweetIDs, 1)
assert.Contains(notif2.TweetIDs, TweetID(1824915465275392037))
assert.Len(notif2.RetweetIDs, 0)
// Simple like: 1 user likes 1 tweet
notif3, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BE-OY688aw"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_LIKE, notif3.Type)
assert.Equal(current_user_id, notif3.UserID)
assert.Equal(UserID(1458284524761075714), notif3.ActionUserID)
assert.Equal(TweetID(1824915465275392037), notif3.ActionTweetID)
assert.Len(notif2.UserIDs, 1)
assert.Contains(notif2.UserIDs, UserID(1458284524761075714))
assert.Len(notif2.TweetIDs, 1)
assert.Contains(notif2.TweetIDs, TweetID(1824915465275392037))
assert.Len(notif2.RetweetIDs, 0)
notif4, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BGLlh8UIQs"]
assert.True(is_ok)
@ -57,14 +70,19 @@ func TestParseNotificationsPage(t *testing.T) {
assert.Equal(current_user_id, notif5.UserID)
assert.Equal(UserID(28815778), notif5.ActionUserID)
// 2 users like 1 tweet
notif6, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BE5ujkCepo"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_LIKE, notif6.Type)
assert.Equal(current_user_id, notif6.UserID)
assert.Equal(UserID(1458284524761075714), notif6.ActionUserID)
assert.Equal(UserID(2694459866), notif6.ActionUserID) // Most recent user
assert.Equal(TweetID(1826778617705115868), notif6.ActionTweetID)
assert.Len(notif6.UserIDs, 2)
assert.Contains(notif6.UserIDs, UserID(1458284524761075714))
assert.Contains(notif6.UserIDs, UserID(2694459866))
assert.Len(notif6.TweetIDs, 1)
assert.Contains(notif6.TweetIDs, TweetID(1826778617705115868))
assert.Len(notif6.RetweetIDs, 0)
notif7, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BGJjUVEd8Y"]
assert.True(is_ok)
@ -74,9 +92,35 @@ func TestParseNotificationsPage(t *testing.T) {
notif8, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BG1nnPGJlQ"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_MENTION, notif8.Type)
assert.Equal(TweetID(1814349573847982537), notif8.ActionTweetID)
// User "liked" your retweet
notif9, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJREwmpDdUTQ"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_LIKE, notif9.Type)
assert.Equal(TweetID(1826778771686392312), notif9.ActionRetweetID)
assert.Equal(TweetID(0), notif9.ActionTweetID) // Tweet is not set
assert.Equal(UserID(1633158398555353096), notif9.ActionUserID)
assert.Len(notif9.TweetIDs, 0)
assert.Len(notif9.UserIDs, 1)
assert.Contains(notif9.UserIDs, UserID(1633158398555353096))
assert.Len(notif9.RetweetIDs, 1)
assert.Contains(notif9.RetweetIDs, TweetID(1826778771686392312))
// Retweet of a retweet
notif10, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJRGACovgUTQ"]
assert.True(is_ok)
assert.Equal(NOTIFICATION_TYPE_RETWEET, notif10.Type)
assert.Equal(TweetID(1827183097382654351), notif10.ActionRetweetID) // the retweet that he retweeted
assert.Equal(TweetID(0), notif10.ActionTweetID)
assert.Len(notif10.UserIDs, 1)
assert.Contains(notif10.UserIDs, UserID(1678546445002059781))
assert.Len(notif10.TweetIDs, 0)
assert.Len(notif10.RetweetIDs, 1)
assert.Contains(notif10.RetweetIDs, TweetID(1827183097382654351))
// Check users
for _, u_id := range []UserID{1458284524761075714, 28815778} {
for _, u_id := range []UserID{1458284524761075714, 28815778, 1633158398555353096} {
_, is_ok := tweet_trove.Users[u_id]
assert.True(is_ok)
}
@ -87,6 +131,12 @@ func TestParseNotificationsPage(t *testing.T) {
assert.True(is_ok)
}
// Check retweets
for _, r_id := range []TweetID{1826778771686392312} {
_, is_ok := tweet_trove.Retweets[r_id]
assert.True(is_ok)
}
// Test cursor-bottom
bottom_cursor := resp.GetCursor()
assert.Equal("DAACDAABCgABFKncQJGVgAQIAAIAAAABCAADSQ3bEQgABIsN6BEACwACAAAAC0FaRkxRSXFNLTJJAAA", bottom_cursor)

40
pkg/scraper/notification.go
View File

@ -5,30 +5,54 @@ type NotificationID string
type NotificationType int
const (
NOTIFICATION_TYPE_LIKE = iota + 1
// ActionUserID is who "liked" it, Action[Re]TweetID is most recent [re]tweet they "liked".
// Can have either many Users "liking" one [re]tweet, or many 1 User "liking" many [re]tweets.
NOTIFICATION_TYPE_LIKE NotificationType = iota + 1
// ActionUserID is who retweeted it, Action[Re]TweetID is your [re]tweet they retweeted.
// Can have either many Users to one [re]tweet, or many [Re]Tweets from 1 User.
NOTIFICATION_TYPE_RETWEET
// ActionUserID is who quote-tweeted you. ActionTweetID is their tweet. ActionRetweet is empty
NOTIFICATION_TYPE_QUOTE_TWEET
// ActionUserID is who replied to you. ActionTweetID is their tweet. ActionRetweet is empty
NOTIFICATION_TYPE_REPLY
// ActionUserID is who followed you. Everything else is empty
NOTIFICATION_TYPE_FOLLOW
// ActionTweetID is their tweet. ActionUserID and ActionRetweetID are empty
NOTIFICATION_TYPE_MENTION
// ActionUserID is who is live. Everything else is empty
NOTIFICATION_TYPE_USER_IS_LIVE
// ActionTweetID is the tweet with the poll. Everything else is empty
NOTIFICATION_TYPE_POLL_ENDED
// Everything is empty
NOTIFICATION_TYPE_LOGIN
// ActionTweetID is the new pinned post. Everything else is empty
NOTIFICATION_TYPE_COMMUNITY_PINNED_POST
// ActionTweetID is the recommended post. Everything else is empty
NOTIFICATION_TYPE_RECOMMENDED_POST
)
type Notification struct {
ID NotificationID `db:"id"`
Type int `db:"type"`
SentAt Timestamp `db:"sent_at"`
SortIndex int64 `db:"sort_index"`
UserID UserID `db:"user_id"` // recipient of the notification
ID NotificationID `db:"id"`
Type NotificationType `db:"type"`
SentAt Timestamp `db:"sent_at"`
SortIndex int64 `db:"sort_index"`
UserID UserID `db:"user_id"` // recipient of the notification
ActionUserID UserID `db:"action_user_id"`
ActionTweetID TweetID `db:"action_tweet_id"`
ActionRetweetID TweetID `db:"action_retweet_id"`
TweetIDs []TweetID
UserIDs []UserID
TweetIDs []TweetID
UserIDs []UserID
RetweetIDs []TweetID
}

2
pkg/scraper/test_responses/notifications/notifications_response_first_page.json
View File

File diff suppressed because one or more lines are too long