Track retweet lists in notifications alongside tweets

pkg/persistence/notification_queries.go

@@ -39,6 +39,16 @@ func (p Profile) SaveNotification(n Notification) {
 			insert into notification_tweets(notification_id, tweet_id) values (?, ?) on conflict do nothing
 		`, n.ID, t_id)
 		if err != nil {
+			fmt.Printf("failed to save notification %#v\n", n)
+			panic(err)
+		}
+	}
+	for _, r_id := range n.RetweetIDs {
+		_, err = tx.Exec(`
+			insert into notification_retweets(notification_id, retweet_id) values (?, ?) on conflict do nothing
+		`, n.ID, r_id)
+		if err != nil {
+			fmt.Printf("failed to save notification %#v\n", n)
 			panic(err)
 		}
 	}
@@ -66,5 +76,9 @@ func (p Profile) GetNotification(id NotificationID) Notification {
 	if err != nil {
 		panic(err)
 	}
+	err = p.DB.Select(&ret.RetweetIDs, `select retweet_id from notification_retweets where notification_id = ?`, id)
+	if err != nil {
+		panic(err)
+	}
 	return ret
 }

pkg/persistence/schema.sql

@@ -422,6 +422,11 @@ create table notification_tweets (rowid integer primary key,
     tweet_id not null references tweets(id),
     unique(notification_id, tweet_id)
 );
+create table notification_retweets (rowid integer primary key,
+    notification_id not null references notifications(id),
+    retweet_id not null references retweets(retweet_id),
+    unique(notification_id, retweet_id)
+);
 create table notification_users (rowid integer primary key,
     notification_id not null references notifications(id),
     user_id not null references users(id),

pkg/persistence/utils_test.go

@@ -409,7 +409,9 @@ func create_dummy_notification() Notification {
 		UserID:        create_stable_user().ID,
 		ActionUserID:  create_stable_user().ID,
 		ActionTweetID: create_stable_tweet().ID,
+		ActionRetweetID: create_stable_retweet().RetweetID,
 		TweetIDs:      []TweetID{create_stable_tweet().ID},
 		UserIDs:       []UserID{create_stable_user().ID},
+		RetweetIDs:    []TweetID{create_stable_retweet().RetweetID},
 	}
 }

pkg/persistence/versions.go

@@ -341,6 +341,11 @@ var MIGRATIONS = []string{
 		    tweet_id not null references tweets(id),
 		    unique(notification_id, tweet_id)
 		);
+		create table notification_retweets (rowid integer primary key,
+		    notification_id not null references notifications(id),
+		    retweet_id not null references retweets(retweet_id),
+		    unique(notification_id, retweet_id)
+		);
 		create table notification_users (rowid integer primary key,
 		    notification_id not null references notifications(id),
 		    user_id not null references users(id),

pkg/scraper/api_types_notifications.go

@@ -92,28 +92,29 @@ func (t *TweetResponse) ToTweetTroveAsNotifications(current_user_id UserID) (Twe
 			} else if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "mentioned") {
 				notification.Type = NOTIFICATION_TYPE_MENTION
 			}
+
 			if entry.Content.Item.Content.Tweet.ID != 0 {
 				notification.ActionTweetID = TweetID(entry.Content.Item.Content.Tweet.ID)
+				notification.TweetIDs = []TweetID{notification.ActionTweetID}
 				notification.ActionUserID = UserID(ret.Tweets[notification.ActionTweetID].UserID)
+			} else if notification.ActionTweetID != TweetID(0) {
+				// Check if it's a tweet or retweet
+				_, is_ok := ret.Retweets[notification.ActionTweetID]
+				if is_ok {
+					// It's a retweet, not a tweet
+					notification.ActionRetweetID = notification.ActionTweetID
+					notification.RetweetIDs = []TweetID{notification.ActionRetweetID}
+					notification.ActionTweetID = TweetID(0) // It's not a tweet
+				} else {
+					// Otherwise, it's a tweet
+					notification.TweetIDs = []TweetID{notification.ActionTweetID}
+				}
 			}
 
-			if entry.Content.Item.Content.Notification.ID != "" {
-				notification.UserIDs = []UserID{}
-				for _, u_id := range entry.Content.Item.Content.Notification.FromUsers {
-					notification.UserIDs = append(notification.UserIDs, UserID(u_id))
-					notification.ActionUserID = UserID(u_id)
-				}
-
-				notification.TweetIDs = []TweetID{}
-				for _, t_id := range entry.Content.Item.Content.Notification.TargetTweets {
-					notification.TweetIDs = append(notification.TweetIDs, TweetID(t_id))
-					notification.ActionTweetID = TweetID(t_id)
-				}
-			}
 			ret.Notifications[notification.ID] = notification
 		}
 	}
-	return ret, err
+	return ret, nil
 }
 
 func ParseSingleNotification(n APINotification) Notification {
@@ -128,8 +129,8 @@ func ParseSingleNotification(n APINotification) Notification {
 
 		n.Message.Text = string(runetext[0:from]) + string(runetext[to:])
 	}
-	// t.Entities.ReplyMentions = strings.TrimSpace(string([]rune(t.FullText)[0:t.DisplayTextRange[0]]))
 
+	// Try to identify the notification type from the notification object itself (not always possible)
 	if strings.HasSuffix(n.Message.Text, "followed you") {
 		ret.Type = NOTIFICATION_TYPE_FOLLOW
 	} else if strings.Contains(n.Message.Text, "liked") {
@@ -139,24 +140,23 @@ func ParseSingleNotification(n APINotification) Notification {
 	} else if strings.Contains(n.Message.Text, "There was a login to your account") {
 		ret.Type = NOTIFICATION_TYPE_LOGIN
 	}
-	// TODO: more types?
 
 	ret.SentAt = TimestampFromUnixMilli(n.TimestampMs)
+
+	// Process UserIDs
 	ret.UserIDs = []UserID{}
 	for _, u := range n.Template.AggregateUserActionsV1.FromUsers {
 		ret.UserIDs = append(ret.UserIDs, UserID(u.User.ID))
+		if ret.ActionUserID == UserID(0) {
+			// "Liking" users are ordered most-recent-first
+			ret.ActionUserID = UserID(u.User.ID)
+		}
 	}
 
+	// If the action has a "target", store it temporarily in ActionTweetID
 	target_objs := n.Template.AggregateUserActionsV1.TargetObjects
 	if len(target_objs) > 0 {
-		if strings.HasSuffix(n.Message.Text, "liked your repost") {
-			// Retweet
-			ret.ActionRetweetID = TweetID(target_objs[0].Tweet.ID)
-		} else {
-			// Normal tweet
 		ret.ActionTweetID = TweetID(target_objs[0].Tweet.ID)
-			ret.TweetIDs = []TweetID{TweetID(target_objs[0].Tweet.ID)}
-		}
 	}
 
 	return ret

pkg/scraper/api_types_notifications_test.go

@@ -29,8 +29,10 @@ func TestParseNotificationsPage(t *testing.T) {
 	notif1, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BFN3re-ZsU"]
 	assert.True(is_ok)
 	assert.Equal(NOTIFICATION_TYPE_LOGIN, notif1.Type)
+	assert.Equal(int64(1723851817578), notif1.SortIndex)
 	assert.Equal(current_user_id, notif1.UserID)
 
+	// Simple retweet: 1 user retweets 1 tweet
 	notif2, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BFaOkNV8aw"]
 	assert.True(is_ok)
 	assert.Equal(NOTIFICATION_TYPE_RETWEET, notif2.Type)
@@ -38,13 +40,24 @@ func TestParseNotificationsPage(t *testing.T) {
 	assert.Equal(UserID(1458284524761075714), notif2.ActionUserID)
 	assert.Equal(TweetID(1824915465275392037), notif2.ActionTweetID)
 	assert.Equal(TimestampFromUnixMilli(1723928739342), notif2.SentAt)
+	assert.Len(notif2.UserIDs, 1)
+	assert.Contains(notif2.UserIDs, UserID(1458284524761075714))
+	assert.Len(notif2.TweetIDs, 1)
+	assert.Contains(notif2.TweetIDs, TweetID(1824915465275392037))
+	assert.Len(notif2.RetweetIDs, 0)
 
+	// Simple like: 1 user likes 1 tweet
 	notif3, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BE-OY688aw"]
 	assert.True(is_ok)
 	assert.Equal(NOTIFICATION_TYPE_LIKE, notif3.Type)
 	assert.Equal(current_user_id, notif3.UserID)
 	assert.Equal(UserID(1458284524761075714), notif3.ActionUserID)
 	assert.Equal(TweetID(1824915465275392037), notif3.ActionTweetID)
+	assert.Len(notif2.UserIDs, 1)
+	assert.Contains(notif2.UserIDs, UserID(1458284524761075714))
+	assert.Len(notif2.TweetIDs, 1)
+	assert.Contains(notif2.TweetIDs, TweetID(1824915465275392037))
+	assert.Len(notif2.RetweetIDs, 0)
 
 	notif4, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BGLlh8UIQs"]
 	assert.True(is_ok)
@@ -57,14 +70,19 @@ func TestParseNotificationsPage(t *testing.T) {
 	assert.Equal(current_user_id, notif5.UserID)
 	assert.Equal(UserID(28815778), notif5.ActionUserID)
 
+	// 2 users like 1 tweet
 	notif6, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BE5ujkCepo"]
 	assert.True(is_ok)
 	assert.Equal(NOTIFICATION_TYPE_LIKE, notif6.Type)
 	assert.Equal(current_user_id, notif6.UserID)
-	assert.Equal(UserID(1458284524761075714), notif6.ActionUserID)
+	assert.Equal(UserID(2694459866), notif6.ActionUserID) // Most recent user
 	assert.Equal(TweetID(1826778617705115868), notif6.ActionTweetID)
+	assert.Len(notif6.UserIDs, 2)
 	assert.Contains(notif6.UserIDs, UserID(1458284524761075714))
 	assert.Contains(notif6.UserIDs, UserID(2694459866))
+	assert.Len(notif6.TweetIDs, 1)
+	assert.Contains(notif6.TweetIDs, TweetID(1826778617705115868))
+	assert.Len(notif6.RetweetIDs, 0)
 
 	notif7, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BGJjUVEd8Y"]
 	assert.True(is_ok)
@@ -74,9 +92,35 @@ func TestParseNotificationsPage(t *testing.T) {
 	notif8, is_ok := tweet_trove.Notifications["FKncQJGVgAQAAAABSQ3bEYsN6BG1nnPGJlQ"]
 	assert.True(is_ok)
 	assert.Equal(NOTIFICATION_TYPE_MENTION, notif8.Type)
+	assert.Equal(TweetID(1814349573847982537), notif8.ActionTweetID)
+
+	// User "liked" your retweet
+	notif9, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJREwmpDdUTQ"]
+	assert.True(is_ok)
+	assert.Equal(NOTIFICATION_TYPE_LIKE, notif9.Type)
+	assert.Equal(TweetID(1826778771686392312), notif9.ActionRetweetID)
+	assert.Equal(TweetID(0), notif9.ActionTweetID) // Tweet is not set
+	assert.Equal(UserID(1633158398555353096), notif9.ActionUserID)
+	assert.Len(notif9.TweetIDs, 0)
+	assert.Len(notif9.UserIDs, 1)
+	assert.Contains(notif9.UserIDs, UserID(1633158398555353096))
+	assert.Len(notif9.RetweetIDs, 1)
+	assert.Contains(notif9.RetweetIDs, TweetID(1826778771686392312))
+
+	// Retweet of a retweet
+	notif10, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJRGACovgUTQ"]
+	assert.True(is_ok)
+	assert.Equal(NOTIFICATION_TYPE_RETWEET, notif10.Type)
+	assert.Equal(TweetID(1827183097382654351), notif10.ActionRetweetID) // the retweet that he retweeted
+	assert.Equal(TweetID(0), notif10.ActionTweetID)
+	assert.Len(notif10.UserIDs, 1)
+	assert.Contains(notif10.UserIDs, UserID(1678546445002059781))
+	assert.Len(notif10.TweetIDs, 0)
+	assert.Len(notif10.RetweetIDs, 1)
+	assert.Contains(notif10.RetweetIDs, TweetID(1827183097382654351))
 
 	// Check users
-	for _, u_id := range []UserID{1458284524761075714, 28815778} {
+	for _, u_id := range []UserID{1458284524761075714, 28815778, 1633158398555353096} {
 		_, is_ok := tweet_trove.Users[u_id]
 		assert.True(is_ok)
 	}
@@ -87,6 +131,12 @@ func TestParseNotificationsPage(t *testing.T) {
 		assert.True(is_ok)
 	}
 
+	// Check retweets
+	for _, r_id := range []TweetID{1826778771686392312} {
+		_, is_ok := tweet_trove.Retweets[r_id]
+		assert.True(is_ok)
+	}
+
 	// Test cursor-bottom
 	bottom_cursor := resp.GetCursor()
 	assert.Equal("DAACDAABCgABFKncQJGVgAQIAAIAAAABCAADSQ3bEQgABIsN6BEACwACAAAAC0FaRkxRSXFNLTJJAAA", bottom_cursor)

pkg/scraper/notification.go

@@ -5,22 +5,45 @@ type NotificationID string
 type NotificationType int
 
 const (
-	NOTIFICATION_TYPE_LIKE = iota + 1
+	// ActionUserID is who "liked" it, Action[Re]TweetID is most recent [re]tweet they "liked".
+	// Can have either many Users "liking" one [re]tweet, or many 1 User "liking" many [re]tweets.
+	NOTIFICATION_TYPE_LIKE NotificationType = iota + 1
+
+	// ActionUserID is who retweeted it, Action[Re]TweetID is your [re]tweet they retweeted.
+	// Can have either many Users to one [re]tweet, or many [Re]Tweets from 1 User.
 	NOTIFICATION_TYPE_RETWEET
+
+	// ActionUserID is who quote-tweeted you.  ActionTweetID is their tweet.  ActionRetweet is empty
 	NOTIFICATION_TYPE_QUOTE_TWEET
+
+	// ActionUserID is who replied to you.  ActionTweetID is their tweet.  ActionRetweet is empty
 	NOTIFICATION_TYPE_REPLY
+
+	// ActionUserID is who followed you.  Everything else is empty
 	NOTIFICATION_TYPE_FOLLOW
+
+	// ActionTweetID is their tweet.  ActionUserID and ActionRetweetID are empty
 	NOTIFICATION_TYPE_MENTION
+
+	// ActionUserID is who is live.  Everything else is empty
 	NOTIFICATION_TYPE_USER_IS_LIVE
+
+	// ActionTweetID is the tweet with the poll.  Everything else is empty
 	NOTIFICATION_TYPE_POLL_ENDED
+
+	// Everything is empty
 	NOTIFICATION_TYPE_LOGIN
+
+	// ActionTweetID is the new pinned post.  Everything else is empty
 	NOTIFICATION_TYPE_COMMUNITY_PINNED_POST
+
+	// ActionTweetID is the recommended post.  Everything else is empty
 	NOTIFICATION_TYPE_RECOMMENDED_POST
 )
 
 type Notification struct {
 	ID        NotificationID   `db:"id"`
-	Type      int            `db:"type"`
+	Type      NotificationType `db:"type"`
 	SentAt    Timestamp        `db:"sent_at"`
 	SortIndex int64            `db:"sort_index"`
 	UserID    UserID           `db:"user_id"` // recipient of the notification
@@ -31,4 +54,5 @@ type Notification struct {
 
 	TweetIDs   []TweetID
 	UserIDs    []UserID
+	RetweetIDs []TweetID
 }