diff --git a/doc/TODO.txt b/doc/TODO.txt index 7609a4b..d64f3a1 100644 --- a/doc/TODO.txt +++ b/doc/TODO.txt @@ -100,15 +100,9 @@ TODO: login-routes-tests - Make the scraper.API object injectable somehow (get rid of singleton pattern) and add tests for login and change-session sequences - Also test profile.ListSessions() -TODO: web-ui-downloading -- web UI needs buttons to trigger a scrape / refresh manually - - timeline - TODO: webserver-session-arg-active-user - make the active user get set on initializing the Application object if a --session flag is given -TODO: webserver-tombstones - TODO: progressive-web-app TODO: paste-twitter-urls-in-search-bar diff --git a/internal/webserver/response_helpers.go b/internal/webserver/response_helpers.go index 2961a5a..fa3d2b2 100644 --- a/internal/webserver/response_helpers.go +++ b/internal/webserver/response_helpers.go @@ -86,13 +86,14 @@ func (app *Application) buffered_render_tweet_page(w http.ResponseWriter, tpl_fi r := renderer{ Funcs: func_map(template.FuncMap{ - "tweet": data.Tweet, - "user": data.User, - "retweet": data.Retweet, - "space": data.Space, - "active_user": app.get_active_user, - "focused_tweet_id": data.FocusedTweetID, - "get_entities": get_entities, + "tweet": data.Tweet, + "user": data.User, + "retweet": data.Retweet, + "space": data.Space, + "active_user": app.get_active_user, + "focused_tweet_id": data.FocusedTweetID, + "get_entities": get_entities, + "get_tombstone_text": get_tombstone_text, }), Filenames: append(partials, get_filepath(tpl_file)), TplName: "base", @@ -120,13 +121,14 @@ func (app *Application) buffered_render_tweet_htmx(w http.ResponseWriter, tpl_na r := renderer{ Funcs: func_map(template.FuncMap{ - "tweet": data.Tweet, - "user": data.User, - "retweet": data.Retweet, - "space": data.Space, - "active_user": app.get_active_user, - "focused_tweet_id": data.FocusedTweetID, - "get_entities": get_entities, + "tweet": data.Tweet, + "user": data.User, + "retweet": data.Retweet, + "space": data.Space, + "active_user": app.get_active_user, + "focused_tweet_id": data.FocusedTweetID, + "get_entities": get_entities, + "get_tombstone_text": get_tombstone_text, }), Filenames: partials, TplName: tpl_name, @@ -190,6 +192,13 @@ func get_entities(text string) []Entity { return ret } +func get_tombstone_text(t scraper.Tweet) string { + if t.TombstoneText != "" { + return t.TombstoneText + } + return t.TombstoneType +} + func func_map(extras template.FuncMap) template.FuncMap { ret := sprig.FuncMap() for i := range extras { diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index 060f085..dc515f2 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -368,6 +368,18 @@ func TestLongTweet(t *testing.T) { } } +func TestTombstoneTweet(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/tweet/31", nil)) + require.Equal(resp.StatusCode, 200) + root, err := html.Parse(resp.Body) + require.NoError(err) + tombstone := cascadia.Query(root, selector(".tweet .tombstone")) + assert.Equal("This Tweet was deleted by the Tweet author", strings.TrimSpace(tombstone.FirstChild.Data)) +} + // Follow and unfollow // ------------------- diff --git a/internal/webserver/static/styles.css b/internal/webserver/static/styles.css index e277106..acca82f 100644 --- a/internal/webserver/static/styles.css +++ b/internal/webserver/static/styles.css @@ -5,10 +5,10 @@ --color-twitter-off-white: #f7f9f9; /* hsv(180, 0.8, 97.6) */ --color-twitter-off-white-dark: #dae5e5; /* hsv(180, 4.8, 89.8) */ --color-outline-gray: #dcdcdc; + --color-twitter-text-gray: #536471; --color-space-purple: #a49bfd; --color-space-purple-outline: #6452fc; - /* const QColor COLOR_OUTLINE_GRAY = QColor(220, 220, 220); const QColor COLOR_TWITTER_BLUE = QColor(27, 149, 224); @@ -557,3 +557,10 @@ ul.dropdown-items { .dropdown-button:focus + .dropdown-items, .dropdown-items:hover { visibility: visible; } +.tombstone { + outline: 1px solid var(--color-outline-gray); + background-color: var(--color-twitter-off-white); + padding: 0.5em 1em; + border-radius: 0.5em; + color: var(--color-twitter-text-gray); +} diff --git a/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl b/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl index 8957c08..e08bc98 100644 --- a/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl +++ b/internal/webserver/tpl/tweet_page_includes/single_tweet.tpl @@ -50,6 +50,11 @@
+ {{if (ne $main_tweet.TombstoneType "")}} +
+ {{(get_tombstone_text $main_tweet)}} +
+ {{end}} {{template "text-with-entities" $main_tweet.Text}} {{range $main_tweet.Images}} diff --git a/pkg/persistence/compound_queries.go b/pkg/persistence/compound_queries.go index db70680..61243c5 100644 --- a/pkg/persistence/compound_queries.go +++ b/pkg/persistence/compound_queries.go @@ -13,6 +13,12 @@ var ( ErrNotInDB = errors.New("not in database") ) +const TWEETS_ALL_SQL_FIELDS = ` + tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, + quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, + ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text, + is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at` + func (p Profile) fill_content(trove *TweetTrove) { if len(trove.Tweets) == 0 { // Empty trove, nothing to fetch @@ -30,10 +36,7 @@ func (p Profile) fill_content(trove *TweetTrove) { if len(quoted_ids) > 0 { var quoted_tweets []Tweet err := p.DB.Select("ed_tweets, ` - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, - ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at + select `+TWEETS_ALL_SQL_FIELDS+` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, quoted_ids...) @@ -215,10 +218,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { where tweets.id = all_replies.id and tweets.in_reply_to_id != 0 ) - select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, - quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, - ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at + select ` + TWEETS_ALL_SQL_FIELDS + ` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid inner join all_replies on tweets.id = all_replies.id @@ -246,10 +246,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { var replies []Tweet stmt, err = p.DB.Preparex( - `select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, - is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at + `select ` + TWEETS_ALL_SQL_FIELDS + ` from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid where in_reply_to_id = ? @@ -287,10 +284,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) { ) ) - select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, - quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, - ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at + select ` + TWEETS_ALL_SQL_FIELDS + ` from top_ids_by_parent left join tweets on tweets.id = top_ids_by_parent.id left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid` @@ -343,76 +337,3 @@ func NewFeed() Feed { TweetTrove: NewTweetTrove(), } } - -// Return the given tweet, all its parent tweets, and a list of conversation threads -func (p Profile) GetUserFeed(id UserID, count int, max_posted_at Timestamp) (Feed, error) { - ret := NewFeed() - - tweet_max_clause := "" - retweet_max_clause := "" - if max_posted_at.Unix() > 0 { - tweet_max_clause = " and posted_at < :max_posted_at " - retweet_max_clause = " and retweeted_at < :max_posted_at " - } - - q := `select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, - is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at, - 0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at, - posted_at order_by - from tweets - left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid - where user_id = :id` + tweet_max_clause + ` - - union - - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, - is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at, - tweet_id, retweet_id, retweeted_by, retweeted_at, - retweeted_at order_by - from retweets - left join tweets on retweets.tweet_id = tweets.id - left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid - where retweeted_by = :id` + retweet_max_clause + ` - - order by order_by desc - limit :limit` - - stmt, err := p.DB.PrepareNamed(q) - if err != nil { - panic(err) - } - - args := map[string]interface{}{ - "id": id, - "limit": count, - "max_posted_at": max_posted_at, - } - var results []struct { - Tweet - Retweet - OrderBy int `db:"order_by"` - } - err = stmt.Select(&results, args) - if err != nil { - panic(err) - } - if len(results) == 0 { - return NewFeed(), ErrEndOfFeed - } - - for _, val := range results { - ret.Tweets[val.Tweet.ID] = val.Tweet - if val.Retweet.RetweetID != 0 { - ret.Retweets[val.Retweet.RetweetID] = val.Retweet - } - ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID}) - } - - p.fill_content(&ret.TweetTrove) - - return ret, nil -} diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index bb276ca..47fda40 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -114,6 +114,20 @@ func TestBuildUserFeedEnd(t *testing.T) { assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END) } +func TestUserFeedWithTombstone(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + profile, err := persistence.LoadProfile("../../sample_data/profile") + require.NoError(err) + + c := persistence.NewUserFeedCursor(UserHandle("Heminator")) + feed, err := profile.NextPage(c) + require.NoError(err) + tombstone_tweet := feed.Tweets[TweetID(31)] + assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author") +} + func TestTweetDetailWithReplies(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/pkg/persistence/compound_ssf_queries.go b/pkg/persistence/compound_ssf_queries.go index f5d1288..3255b46 100644 --- a/pkg/persistence/compound_ssf_queries.go +++ b/pkg/persistence/compound_ssf_queries.go @@ -360,10 +360,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { where_clause := "where " + strings.Join(where_clauses, " and ") q := `select * from ( - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, - is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at, + select ` + TWEETS_ALL_SQL_FIELDS + `, 0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at, posted_at chrono, user_id by_user_id from tweets @@ -374,10 +371,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) { union select * from ( - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, - mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type, - is_expandable, - is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at, + select ` + TWEETS_ALL_SQL_FIELDS + `, tweet_id, retweet_id, retweeted_by, retweeted_at, retweeted_at chrono, retweeted_by by_user_id from retweets diff --git a/pkg/scraper/tweet.go b/pkg/scraper/tweet.go index 3638d81..86405c8 100644 --- a/pkg/scraper/tweet.go +++ b/pkg/scraper/tweet.go @@ -70,6 +70,7 @@ type Tweet struct { SpaceID SpaceID `db:"space_id"` TombstoneType string `db:"tombstone_type"` + TombstoneText string `db:"tombstone_text"` IsStub bool `db:"is_stub"` IsContentDownloaded bool `db:"is_content_downloaded"`