Show tombstones

doc/TODO.txt

@@ -100,15 +100,9 @@ TODO: login-routes-tests
 - Make the scraper.API object injectable somehow (get rid of singleton pattern) and add tests for login and change-session sequences
 - Also test profile.ListSessions()
 
-TODO: web-ui-downloading
-- web UI needs buttons to trigger a scrape / refresh manually
-	- timeline
-
 TODO: webserver-session-arg-active-user
 - make the active user get set on initializing the Application object if a --session flag is given
 
-TODO: webserver-tombstones
-
 TODO: progressive-web-app
 
 TODO: paste-twitter-urls-in-search-bar

internal/webserver/response_helpers.go

@@ -86,13 +86,14 @@ func (app *Application) buffered_render_tweet_page(w http.ResponseWriter, tpl_fi
 
 	r := renderer{
 		Funcs: func_map(template.FuncMap{
-			"tweet":            data.Tweet,
+			"tweet":              data.Tweet,
-			"user":             data.User,
+			"user":               data.User,
-			"retweet":          data.Retweet,
+			"retweet":            data.Retweet,
-			"space":            data.Space,
+			"space":              data.Space,
-			"active_user":      app.get_active_user,
+			"active_user":        app.get_active_user,
-			"focused_tweet_id": data.FocusedTweetID,
+			"focused_tweet_id":   data.FocusedTweetID,
-			"get_entities":     get_entities,
+			"get_entities":       get_entities,
+			"get_tombstone_text": get_tombstone_text,
 		}),
 		Filenames: append(partials, get_filepath(tpl_file)),
 		TplName:   "base",
@@ -120,13 +121,14 @@ func (app *Application) buffered_render_tweet_htmx(w http.ResponseWriter, tpl_na
 
 	r := renderer{
 		Funcs: func_map(template.FuncMap{
-			"tweet":            data.Tweet,
+			"tweet":              data.Tweet,
-			"user":             data.User,
+			"user":               data.User,
-			"retweet":          data.Retweet,
+			"retweet":            data.Retweet,
-			"space":            data.Space,
+			"space":              data.Space,
-			"active_user":      app.get_active_user,
+			"active_user":        app.get_active_user,
-			"focused_tweet_id": data.FocusedTweetID,
+			"focused_tweet_id":   data.FocusedTweetID,
-			"get_entities":     get_entities,
+			"get_entities":       get_entities,
+			"get_tombstone_text": get_tombstone_text,
 		}),
 		Filenames: partials,
 		TplName:   tpl_name,
@@ -190,6 +192,13 @@ func get_entities(text string) []Entity {
 	return ret
 }
 
+func get_tombstone_text(t scraper.Tweet) string {
+	if t.TombstoneText != "" {
+		return t.TombstoneText
+	}
+	return t.TombstoneType
+}
+
 func func_map(extras template.FuncMap) template.FuncMap {
 	ret := sprig.FuncMap()
 	for i := range extras {

internal/webserver/server_test.go

@@ -368,6 +368,18 @@ func TestLongTweet(t *testing.T) {
 	}
 }
 
+func TestTombstoneTweet(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/tweet/31", nil))
+	require.Equal(resp.StatusCode, 200)
+	root, err := html.Parse(resp.Body)
+	require.NoError(err)
+	tombstone := cascadia.Query(root, selector(".tweet .tombstone"))
+	assert.Equal("This Tweet was deleted by the Tweet author", strings.TrimSpace(tombstone.FirstChild.Data))
+}
+
 // Follow and unfollow
 // -------------------
 

internal/webserver/static/styles.css

@@ -5,10 +5,10 @@
     --color-twitter-off-white: #f7f9f9; /* hsv(180, 0.8, 97.6) */
     --color-twitter-off-white-dark: #dae5e5; /* hsv(180, 4.8, 89.8) */
     --color-outline-gray: #dcdcdc;
+    --color-twitter-text-gray: #536471;
 
     --color-space-purple: #a49bfd;
     --color-space-purple-outline: #6452fc;
-
 /*
 const QColor COLOR_OUTLINE_GRAY = QColor(220, 220, 220);
 const QColor COLOR_TWITTER_BLUE = QColor(27, 149, 224);
@@ -557,3 +557,10 @@ ul.dropdown-items {
 .dropdown-button:focus + .dropdown-items, .dropdown-items:hover {
     visibility: visible;
 }
+.tombstone {
+    outline: 1px solid var(--color-outline-gray);
+    background-color: var(--color-twitter-off-white);
+    padding: 0.5em 1em;
+    border-radius: 0.5em;
+    color: var(--color-twitter-text-gray);
+}

internal/webserver/tpl/tweet_page_includes/single_tweet.tpl

@@ -50,6 +50,11 @@
     </span>
     <span class="vertical-container-1">
       <div class="tweet-content">
+        {{if (ne $main_tweet.TombstoneType "")}}
+          <div class="tombstone">
+            {{(get_tombstone_text $main_tweet)}}
+          </div>
+        {{end}}
         {{template "text-with-entities" $main_tweet.Text}}
         {{range $main_tweet.Images}}
           <img src="/content/images/{{.LocalFilename}}" style="max-width: 45%"/>

pkg/persistence/compound_queries.go

@@ -13,6 +13,12 @@ var (
 	ErrNotInDB   = errors.New("not in database")
 )
 
+const TWEETS_ALL_SQL_FIELDS = `
+		tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
+		quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
+		ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text,
+		is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at`
+
 func (p Profile) fill_content(trove *TweetTrove) {
 	if len(trove.Tweets) == 0 {
 		// Empty trove, nothing to fetch
@@ -30,10 +36,7 @@ func (p Profile) fill_content(trove *TweetTrove) {
 	if len(quoted_ids) > 0 {
 		var quoted_tweets []Tweet
 		err := p.DB.Select(&quoted_tweets, `
-		     select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
+		     select `+TWEETS_ALL_SQL_FIELDS+`
-		            mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
-		            ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
-		            is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
 		       from tweets
 		  left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
 		      where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, quoted_ids...)
@@ -215,10 +218,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
 		         where tweets.id = all_replies.id and tweets.in_reply_to_id != 0
 		        )
 
-		 select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
+		 select ` + TWEETS_ALL_SQL_FIELDS + `
-		        quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
-		        ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
-	            is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
 	       from tweets
 	  left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
 	 inner join all_replies on tweets.id = all_replies.id
@@ -246,10 +246,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
 
 	var replies []Tweet
 	stmt, err = p.DB.Preparex(
-		`select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
+		`select ` + TWEETS_ALL_SQL_FIELDS + `
-	            mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
-	            is_expandable,
-	            is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
 	       from tweets
 	  left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
 	      where in_reply_to_id = ?
@@ -287,10 +284,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
 		                )
 		           )
 
-		    select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
+		    select ` + TWEETS_ALL_SQL_FIELDS + `
-		           quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
-		           ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
-		           is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
 		      from top_ids_by_parent
 		 left join tweets on tweets.id = top_ids_by_parent.id
 		 left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid`
@@ -343,76 +337,3 @@ func NewFeed() Feed {
 		TweetTrove: NewTweetTrove(),
 	}
 }
-
-// Return the given tweet, all its parent tweets, and a list of conversation threads
-func (p Profile) GetUserFeed(id UserID, count int, max_posted_at Timestamp) (Feed, error) {
-	ret := NewFeed()
-
-	tweet_max_clause := ""
-	retweet_max_clause := ""
-	if max_posted_at.Unix() > 0 {
-		tweet_max_clause = " and posted_at < :max_posted_at "
-		retweet_max_clause = " and retweeted_at < :max_posted_at "
-	}
-
-	q := `select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
-               mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
-               is_expandable,
-               is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
-               0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at,
-               posted_at order_by
-          from tweets
-     left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
-         where user_id = :id` + tweet_max_clause + `
-
-         union
-
-        select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
-               mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
-               is_expandable,
-               is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
-               tweet_id, retweet_id, retweeted_by, retweeted_at,
-               retweeted_at order_by
-          from retweets
-     left join tweets on retweets.tweet_id = tweets.id
-     left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
-         where retweeted_by = :id` + retweet_max_clause + `
-
-         order by order_by desc
-         limit :limit`
-
-	stmt, err := p.DB.PrepareNamed(q)
-	if err != nil {
-		panic(err)
-	}
-
-	args := map[string]interface{}{
-		"id":            id,
-		"limit":         count,
-		"max_posted_at": max_posted_at,
-	}
-	var results []struct {
-		Tweet
-		Retweet
-		OrderBy int `db:"order_by"`
-	}
-	err = stmt.Select(&results, args)
-	if err != nil {
-		panic(err)
-	}
-	if len(results) == 0 {
-		return NewFeed(), ErrEndOfFeed
-	}
-
-	for _, val := range results {
-		ret.Tweets[val.Tweet.ID] = val.Tweet
-		if val.Retweet.RetweetID != 0 {
-			ret.Retweets[val.Retweet.RetweetID] = val.Retweet
-		}
-		ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID})
-	}
-
-	p.fill_content(&ret.TweetTrove)
-
-	return ret, nil
-}

pkg/persistence/compound_queries_test.go

@@ -114,6 +114,20 @@ func TestBuildUserFeedEnd(t *testing.T) {
 	assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END)
 }
 
+func TestUserFeedWithTombstone(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	profile, err := persistence.LoadProfile("../../sample_data/profile")
+	require.NoError(err)
+
+	c := persistence.NewUserFeedCursor(UserHandle("Heminator"))
+	feed, err := profile.NextPage(c)
+	require.NoError(err)
+	tombstone_tweet := feed.Tweets[TweetID(31)]
+	assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author")
+}
+
 func TestTweetDetailWithReplies(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)

pkg/persistence/compound_ssf_queries.go

@@ -360,10 +360,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
 	where_clause := "where " + strings.Join(where_clauses, " and ")
 
 	q := `select * from (
-	select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
+	select ` + TWEETS_ALL_SQL_FIELDS + `,
-           mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
-           is_expandable,
-           is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
            0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at,
            posted_at chrono, user_id by_user_id
       from tweets
@@ -374,10 +371,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
      union
 
     select * from (
-    select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
+    select ` + TWEETS_ALL_SQL_FIELDS + `,
-           mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
-           is_expandable,
-           is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
            tweet_id, retweet_id, retweeted_by, retweeted_at,
            retweeted_at chrono, retweeted_by by_user_id
       from retweets

pkg/scraper/tweet.go

@@ -70,6 +70,7 @@ type Tweet struct {
 	SpaceID SpaceID `db:"space_id"`
 
 	TombstoneType string `db:"tombstone_type"`
+	TombstoneText string `db:"tombstone_text"`
 	IsStub        bool   `db:"is_stub"`
 
 	IsContentDownloaded   bool      `db:"is_content_downloaded"`