me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Show tombstones

Browse Source
This commit is contained in:
Alessio 2023-08-29 12:27:31 -03:00
parent 8aca7d4ebe
commit b2df94f041
9 changed files with 75 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/TODO.txt
View File

@ -100,15 +100,9 @@ TODO: login-routes-tests
- Make the scraper.API object injectable somehow (get rid of singleton pattern) and add tests for login and change-session sequences
- Also test profile.ListSessions()
TODO: web-ui-downloading
- web UI needs buttons to trigger a scrape / refresh manually
- timeline
TODO: webserver-session-arg-active-user
- make the active user get set on initializing the Application object if a --session flag is given
TODO: webserver-tombstones
TODO: progressive-web-app
TODO: paste-twitter-urls-in-search-bar

37
internal/webserver/response_helpers.go
View File

@ -86,13 +86,14 @@ func (app *Application) buffered_render_tweet_page(w http.ResponseWriter, tpl_fi
r := renderer{
Funcs: func_map(template.FuncMap{
"tweet": data.Tweet,
"user": data.User,
"retweet": data.Retweet,
"space": data.Space,
"active_user": app.get_active_user,
"focused_tweet_id": data.FocusedTweetID,
"get_entities": get_entities,
"tweet": data.Tweet,
"user": data.User,
"retweet": data.Retweet,
"space": data.Space,
"active_user": app.get_active_user,
"focused_tweet_id": data.FocusedTweetID,
"get_entities": get_entities,
"get_tombstone_text": get_tombstone_text,
}),
Filenames: append(partials, get_filepath(tpl_file)),
TplName: "base",
@ -120,13 +121,14 @@ func (app *Application) buffered_render_tweet_htmx(w http.ResponseWriter, tpl_na
r := renderer{
Funcs: func_map(template.FuncMap{
"tweet": data.Tweet,
"user": data.User,
"retweet": data.Retweet,
"space": data.Space,
"active_user": app.get_active_user,
"focused_tweet_id": data.FocusedTweetID,
"get_entities": get_entities,
"tweet": data.Tweet,
"user": data.User,
"retweet": data.Retweet,
"space": data.Space,
"active_user": app.get_active_user,
"focused_tweet_id": data.FocusedTweetID,
"get_entities": get_entities,
"get_tombstone_text": get_tombstone_text,
}),
Filenames: partials,
TplName: tpl_name,
@ -190,6 +192,13 @@ func get_entities(text string) []Entity {
return ret
}
func get_tombstone_text(t scraper.Tweet) string {
if t.TombstoneText != "" {
return t.TombstoneText
}
return t.TombstoneType
}
func func_map(extras template.FuncMap) template.FuncMap {
ret := sprig.FuncMap()
for i := range extras {

12
internal/webserver/server_test.go
View File

@ -368,6 +368,18 @@ func TestLongTweet(t *testing.T) {
}
}
func TestTombstoneTweet(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
resp := do_request(httptest.NewRequest("GET", "/tweet/31", nil))
require.Equal(resp.StatusCode, 200)
root, err := html.Parse(resp.Body)
require.NoError(err)
tombstone := cascadia.Query(root, selector(".tweet .tombstone"))
assert.Equal("This Tweet was deleted by the Tweet author", strings.TrimSpace(tombstone.FirstChild.Data))
}
// Follow and unfollow
// -------------------

9
internal/webserver/static/styles.css
View File

@ -5,10 +5,10 @@
--color-twitter-off-white: #f7f9f9; /* hsv(180, 0.8, 97.6) */
--color-twitter-off-white-dark: #dae5e5; /* hsv(180, 4.8, 89.8) */
--color-outline-gray: #dcdcdc;
--color-twitter-text-gray: #536471;
--color-space-purple: #a49bfd;
--color-space-purple-outline: #6452fc;
/*
const QColor COLOR_OUTLINE_GRAY = QColor(220, 220, 220);
const QColor COLOR_TWITTER_BLUE = QColor(27, 149, 224);
@ -557,3 +557,10 @@ ul.dropdown-items {
.dropdown-button:focus + .dropdown-items, .dropdown-items:hover {
visibility: visible;
}
.tombstone {
outline: 1px solid var(--color-outline-gray);
background-color: var(--color-twitter-off-white);
padding: 0.5em 1em;
border-radius: 0.5em;
color: var(--color-twitter-text-gray);
}

5
internal/webserver/tpl/tweet_page_includes/single_tweet.tpl
View File

@ -50,6 +50,11 @@
</span>
<span class="vertical-container-1">
<div class="tweet-content">
{{if (ne $main_tweet.TombstoneType "")}}
<div class="tombstone">
{{(get_tombstone_text $main_tweet)}}
</div>
{{end}}
{{template "text-with-entities" $main_tweet.Text}}
{{range $main_tweet.Images}}
<img src="/content/images/{{.LocalFilename}}" style="max-width: 45%"/>

99
pkg/persistence/compound_queries.go
View File

@ -13,6 +13,12 @@ var (
ErrNotInDB = errors.New("not in database")
)
const TWEETS_ALL_SQL_FIELDS = `
tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
ifnull(tombstone_types.short_name, "") tombstone_type, ifnull(tombstone_types.tombstone_text, "") tombstone_text,
is_expandable, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at`
func (p Profile) fill_content(trove *TweetTrove) {
if len(trove.Tweets) == 0 {
// Empty trove, nothing to fetch
@ -30,10 +36,7 @@ func (p Profile) fill_content(trove *TweetTrove) {
if len(quoted_ids) > 0 {
var quoted_tweets []Tweet
err := p.DB.Select(&quoted_tweets, `
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
select `+TWEETS_ALL_SQL_FIELDS+`
from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where id in (`+strings.Repeat("?,", len(quoted_ids)-1)+`?)`, quoted_ids...)
@ -215,10 +218,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
where tweets.id = all_replies.id and tweets.in_reply_to_id != 0
)
select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
select ` + TWEETS_ALL_SQL_FIELDS + `
from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
inner join all_replies on tweets.id = all_replies.id
@ -246,10 +246,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
var replies []Tweet
stmt, err = p.DB.Preparex(
`select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
`select ` + TWEETS_ALL_SQL_FIELDS + `
from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where in_reply_to_id = ?
@ -287,10 +284,7 @@ func (p Profile) GetTweetDetail(id TweetID) (TweetDetailView, error) {
)
)
select tweets.id id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id,
ifnull(tombstone_types.short_name, "") tombstone_type, is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at
select ` + TWEETS_ALL_SQL_FIELDS + `
from top_ids_by_parent
left join tweets on tweets.id = top_ids_by_parent.id
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid`
@ -343,76 +337,3 @@ func NewFeed() Feed {
TweetTrove: NewTweetTrove(),
}
}
// Return the given tweet, all its parent tweets, and a list of conversation threads
func (p Profile) GetUserFeed(id UserID, count int, max_posted_at Timestamp) (Feed, error) {
ret := NewFeed()
tweet_max_clause := ""
retweet_max_clause := ""
if max_posted_at.Unix() > 0 {
tweet_max_clause = " and posted_at < :max_posted_at "
retweet_max_clause = " and retweeted_at < :max_posted_at "
}
q := `select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at,
posted_at order_by
from tweets
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where user_id = :id` + tweet_max_clause + `
union
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
tweet_id, retweet_id, retweeted_by, retweeted_at,
retweeted_at order_by
from retweets
left join tweets on retweets.tweet_id = tweets.id
left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid
where retweeted_by = :id` + retweet_max_clause + `
order by order_by desc
limit :limit`
stmt, err := p.DB.PrepareNamed(q)
if err != nil {
panic(err)
}
args := map[string]interface{}{
"id": id,
"limit": count,
"max_posted_at": max_posted_at,
}
var results []struct {
Tweet
Retweet
OrderBy int `db:"order_by"`
}
err = stmt.Select(&results, args)
if err != nil {
panic(err)
}
if len(results) == 0 {
return NewFeed(), ErrEndOfFeed
}
for _, val := range results {
ret.Tweets[val.Tweet.ID] = val.Tweet
if val.Retweet.RetweetID != 0 {
ret.Retweets[val.Retweet.RetweetID] = val.Retweet
}
ret.Items = append(ret.Items, FeedItem{TweetID: val.Tweet.ID, RetweetID: val.Retweet.RetweetID})
}
p.fill_content(&ret.TweetTrove)
return ret, nil
}

14
pkg/persistence/compound_queries_test.go
View File

@ -114,6 +114,20 @@ func TestBuildUserFeedEnd(t *testing.T) {
assert.Equal(feed.CursorBottom.CursorPosition, persistence.CURSOR_END)
}
func TestUserFeedWithTombstone(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
profile, err := persistence.LoadProfile("../../sample_data/profile")
require.NoError(err)
c := persistence.NewUserFeedCursor(UserHandle("Heminator"))
feed, err := profile.NextPage(c)
require.NoError(err)
tombstone_tweet := feed.Tweets[TweetID(31)]
assert.Equal(tombstone_tweet.TombstoneText, "This Tweet was deleted by the Tweet author")
}
func TestTweetDetailWithReplies(t *testing.T) {
require := require.New(t)
assert := assert.New(t)

10
pkg/persistence/compound_ssf_queries.go
View File

@ -360,10 +360,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
where_clause := "where " + strings.Join(where_clauses, " and ")
q := `select * from (
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
select ` + TWEETS_ALL_SQL_FIELDS + `,
0 tweet_id, 0 retweet_id, 0 retweeted_by, 0 retweeted_at,
posted_at chrono, user_id by_user_id
from tweets
@ -374,10 +371,7 @@ func (p Profile) NextPage(c Cursor) (Feed, error) {
union
select * from (
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(space_id, '') space_id, ifnull(tombstone_types.short_name, "") tombstone_type,
is_expandable,
is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at,
select ` + TWEETS_ALL_SQL_FIELDS + `,
tweet_id, retweet_id, retweeted_by, retweeted_at,
retweeted_at chrono, retweeted_by by_user_id
from retweets

1
pkg/scraper/tweet.go
View File

@ -70,6 +70,7 @@ type Tweet struct {
SpaceID SpaceID `db:"space_id"`
TombstoneType string `db:"tombstone_type"`
TombstoneText string `db:"tombstone_text"`
IsStub bool `db:"is_stub"`
IsContentDownloaded bool `db:"is_content_downloaded"`