Add Lists pages

- List index page
- List detail pages: Feed and Users
- Improve styling for page headings (e.g., Search, Follows, list title, etc)

internal/webserver/handler_lists.go

@@ -1,22 +1,28 @@
 package webserver
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
 
-	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
+	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
 )
 
 type ListData struct {
-	Title         string
-	HeaderUserID  scraper.UserID
-	HeaderTweetID scraper.TweetID
-	UserIDs       []scraper.UserID
+	List
+	Feed      persistence.Feed
+	UserIDs   []UserID
+	ActiveTab string
 }
 
-func NewListData(users []scraper.User) (ListData, scraper.TweetTrove) {
-	trove := scraper.NewTweetTrove()
+func NewListData(users []User) (ListData, TweetTrove) {
+	trove := NewTweetTrove()
 	data := ListData{
-		UserIDs: []scraper.UserID{},
+		UserIDs: []UserID{},
 	}
 	for _, u := range users {
 		trove.Users[u.ID] = u
@@ -25,19 +31,139 @@ func NewListData(users []scraper.User) (ListData, scraper.TweetTrove) {
 	return data, trove
 }
 
+func (app *Application) ListDetailFeed(w http.ResponseWriter, r *http.Request) {
+	list := get_list_from_context(r.Context())
+
+	c := persistence.NewListCursor(list.ID)
+	err := parse_cursor_value(&c, r)
+	if err != nil {
+		app.error_400_with_message(w, "invalid cursor (must be a number)")
+		return
+	}
+	feed, err := app.Profile.NextPage(c, app.ActiveUser.ID)
+	if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) {
+		panic(err)
+	}
+	if r.Header.Get("HX-Request") == "true" && c.CursorPosition == persistence.CURSOR_MIDDLE {
+		// It's a Show More request
+		app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, feed)
+	} else {
+		app.buffered_render_page(
+			w,
+			"tpl/list.tpl",
+			PageGlobalData{TweetTrove: feed.TweetTrove},
+			ListData{Feed: feed, List: list, ActiveTab: "feed"},
+		)
+	}
+}
+
+func (app *Application) ListDetailUsers(w http.ResponseWriter, r *http.Request) {
+	list := get_list_from_context(r.Context())
+	users := app.Profile.GetListUsers(list.ID)
+
+	data, trove := NewListData(users)
+	data.List = list
+	data.ActiveTab = "users"
+	app.buffered_render_page(w, "tpl/list.tpl", PageGlobalData{TweetTrove: trove}, data)
+}
+
+func (app *Application) ListDetail(w http.ResponseWriter, r *http.Request) {
+	app.traceLog.Printf("'ListDetail' handler (path: %q)", r.URL.Path)
+	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
+
+	if len(parts) == 1 && parts[0] == "" {
+		// No further path; just show the feed
+		app.ListDetailFeed(w, r)
+	}
+
+	switch parts[0] {
+	case "users":
+		app.ListDetailUsers(w, r)
+	case "add_user":
+		app.ListAddUser(w, r)
+	case "remove_user":
+		app.ListRemoveUser(w, r)
+	default:
+		app.error_404(w)
+	}
+}
+
+func (app *Application) ListAddUser(w http.ResponseWriter, r *http.Request) {
+	handle := r.URL.Query().Get("user_handle")
+	if handle[0] == '@' {
+		handle = handle[1:]
+	}
+	user, err := app.Profile.GetUserByHandle(UserHandle(handle))
+	if err != nil {
+		app.error_400_with_message(w, "Fetch user: "+err.Error())
+		return
+	}
+	list := get_list_from_context(r.Context())
+	app.Profile.SaveListUser(list.ID, user.ID)
+	http.Redirect(w, r, fmt.Sprintf("/lists/%d", list.ID), 302)
+}
+
+func (app *Application) ListRemoveUser(w http.ResponseWriter, r *http.Request) {
+	handle := r.URL.Query().Get("user_handle")
+	if handle[0] == '@' {
+		handle = handle[1:]
+	}
+	user, err := app.Profile.GetUserByHandle(UserHandle(handle))
+	if err != nil {
+		app.error_400_with_message(w, "Fetch user: "+err.Error())
+		return
+	}
+	list := get_list_from_context(r.Context())
+	app.Profile.DeleteListUser(list.ID, user.ID)
+	http.Redirect(w, r, fmt.Sprintf("/lists/%d", list.ID), 302)
+}
+
 func (app *Application) Lists(w http.ResponseWriter, r *http.Request) {
 	app.traceLog.Printf("'Lists' handler (path: %q)", r.URL.Path)
 
-	var users []scraper.User
-	err := app.Profile.DB.Select(&users, `
-		select id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified,
-	           is_banned, is_deleted, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
-	           pinned_tweet_id, is_content_downloaded, is_followed
-	      from users
-	     where is_followed = 1`)
-	panic_if(err)
+	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
 
-	data, trove := NewListData(users)
-	data.Title = "Offline Follows"
-	app.buffered_render_page(w, "tpl/list.tpl", PageGlobalData{TweetTrove: trove}, data)
+	// List detail
+	if parts[0] != "" { // If there's an ID param
+		_list_id, err := strconv.Atoi(parts[0])
+		if err != nil {
+			app.error_400_with_message(w, "List ID must be a number")
+			return
+		}
+		// XXX: Check that the list exists
+		// Need to modify signature to return an error, because it might be ErrNoRows
+		list := app.Profile.GetListById(ListID(_list_id))
+		req_with_ctx := r.WithContext(add_list_to_context(r.Context(), list))
+		http.StripPrefix(fmt.Sprintf("/%d", list.ID), http.HandlerFunc(app.ListDetail)).ServeHTTP(w, req_with_ctx)
+		return
+	}
+
+	// List index
+	lists := app.Profile.GetAllLists()
+	trove := NewTweetTrove()
+	for _, l := range lists {
+		for _, u := range l.Users {
+			trove.Users[u.ID] = u
+		}
+	}
+	app.buffered_render_page(
+		w,
+		"tpl/list_of_lists.tpl",
+		PageGlobalData{TweetTrove: trove},
+		lists,
+	)
+}
+
+const LIST_KEY = key("list") // type `key` is defined in "handler_tweet_detail"
+
+func add_list_to_context(ctx context.Context, list List) context.Context {
+	return context.WithValue(ctx, LIST_KEY, list)
+}
+
+func get_list_from_context(ctx context.Context) List {
+	list, is_ok := ctx.Value(LIST_KEY).(List)
+	if !is_ok {
+		panic("List not found in context")
+	}
+	return list
 }

internal/webserver/server.go

@@ -124,7 +124,7 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case "search":
 		http.StripPrefix("/search", http.HandlerFunc(app.Search)).ServeHTTP(w, r)
 	case "lists":
-		app.Lists(w, r)
+		http.StripPrefix("/lists", http.HandlerFunc(app.Lists)).ServeHTTP(w, r)
 	case "messages":
 		http.StripPrefix("/messages", http.HandlerFunc(app.Messages)).ServeHTTP(w, r)
 	default:

internal/webserver/server_test.go

@@ -590,12 +590,39 @@ func TestStaticFileNonexistent(t *testing.T) {
 // Lists
 // -----
 
-func TestLists(t *testing.T) {
-	assert := assert.New(t)
+func TestListsIndex(t *testing.T) {
+	require := require.New(t)
 	resp := do_request(httptest.NewRequest("GET", "/lists", nil))
+	require.Equal(resp.StatusCode, 200)
 	root, err := html.Parse(resp.Body)
-	assert.NoError(err)
+	require.NoError(err)
+
+	// Check that there's 2 Lists
+	assert.Len(t, cascadia.QueryAll(root, selector(".users-list-preview")), 2)
+}
+
+func TestListDetail(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	// Users
+	resp := do_request(httptest.NewRequest("GET", "/lists/1/users", nil))
+	require.Equal(resp.StatusCode, 200)
+	root, err := html.Parse(resp.Body)
+	require.NoError(err)
 	assert.Len(cascadia.QueryAll(root, selector(".users-list-container .author-info")), 5)
+
+	// Feed
+	resp1 := do_request(httptest.NewRequest("GET", "/lists/2", nil))
+	require.Equal(resp1.StatusCode, 200)
+	root1, err := html.Parse(resp1.Body)
+	require.NoError(err)
+	assert.Len(cascadia.QueryAll(root1, selector(".timeline > .tweet")), 3)
+}
+
+func TestListDetailInvalidId(t *testing.T) {
+	resp := do_request(httptest.NewRequest("GET", "/lists/asd", nil))
+	require.Equal(t, resp.StatusCode, 400)
 }
 
 // Messages

internal/webserver/static/styles.css

@@ -51,6 +51,11 @@ input, select {
     border-radius: 0.5em;
 }
 
+h1 {
+    margin: 0.5em 0;
+    text-align: center;
+}
+
 .server-error-msg {
     position: fixed;
     margin: 6em 0;
@@ -267,7 +272,7 @@ h3 {
     width: 3em;
     height: 3em;
     display: inline;
-    border: 1px solid var(--color-outline-gray);
+    border: 0.1em solid var(--color-outline-gray);
 }
 
 .user-header {
@@ -657,9 +662,46 @@ ul.space-participants-list li {
     filter: invert(20%) sepia(97%) saturate(4383%) hue-rotate(321deg) brightness(101%) contrast(95%);
 }
 
+.users-list-previews {
+    border-color: var(--color-twitter-off-white-dark);
+    border-top-style: double;
+    border-width: 4px;
+}
+
+.users-list-preview {
+    cursor: pointer;
+    display: flex;
+    font-size: 1.5em;
+    border-color: var(--color-twitter-off-white-dark);
+    border-bottom-style: solid;
+    border-width: 1px;
+    padding: 0.5em 1em;
+    align-items: center;
+}
+.users-list-preview span.num-users {
+    margin-left: 1em;
+    color: var(--color-twitter-text-gray);
+}
+.users-list-preview .first-N-profile-images {
+    display: flex;
+    align-items: flex-end;
+    margin-left: 1.5em;
+}
+.users-list-preview .first-N-profile-images a {
+    margin-right: -1.2em;
+    line-height: 0; /* TODO: This is duplicated from `.author-info a` and possibly others */
+}
+
+.users-list-preview .first-N-profile-images .ellipsis {
+    margin-left: 1.5em;
+}
+
 .users-list-container {
     display: flex;
     flex-direction: column;
+    border-color: var(--color-twitter-off-white-dark);
+    border-top-style: double;
+    border-width: 4px;
 }
 .users-list-container .author-info .profile-image {
     width: 4em;

internal/webserver/tpl/list.tpl

@@ -1,15 +1,31 @@
-{{define "title"}}{{.Title}}{{end}}
+{{define "title"}}{{.List.Name}}{{end}}
 
 {{define "main"}}
-  {{if .HeaderUserID}}
-    {{template "user-header" (user .HeaderUserID)}}
-  {{else if .HeaderTweetID}}
-    {{template "tweet" (tweet .HeaderTweetID)}}
-  {{end}}
+  <div class="list-feed-header">
+    <h1>{{.List.Name}}</h1>
 
-  <h3>
-    {{.Title}}
-  </h3>
+    <div class="row tabs-container">
+      <a class="tab unstyled-link {{if (eq .ActiveTab "feed")}}active-tab{{end}}" href="/lists/{{.List.ID}}">
+        <span class="tab-inner">Feed</span>
+      </a>
+      <a class="tab unstyled-link {{if (eq .ActiveTab "users")}}active-tab{{end}}" href="/lists/{{.List.ID}}/users">
+        <span class="tab-inner">Users</span>
+      </a>
+    </div>
+  </div>
+
+  {{if (eq .ActiveTab "feed")}}
+    <div class="timeline list-feed-timeline">
+      {{template "timeline" .Feed}}
+    </div>
+  {{else}}
+    <div class="add-users-container">
+      <form action="/lists/{{.List.ID}}/add_user">
+        <input type="text" name="user_handle" placeholder="@some_user_handle" />
+        <input type="submit" value="Add user" />
+      </form>
+    </div>
 
     {{template "list" .UserIDs}}
   {{end}}
+{{end}}

internal/webserver/tpl/search.tpl

@@ -2,7 +2,7 @@
 
 {{define "main"}}
   <div class="search-header">
-    <h2 style="text-align: center">Search results: {{.SearchText}}</h2>
+    <h1>Search results: {{.SearchText}}</h1>
 
     <div class="row tabs-container">
       <a

pkg/persistence/compound_ssf_queries.go

@@ -129,11 +129,12 @@ type Cursor struct {
 
 	// Search params
 	Keywords              []string
-	FromUserHandle        scraper.UserHandle
-	RetweetedByUserHandle scraper.UserHandle
-	ByUserHandle          scraper.UserHandle
-	ToUserHandles         []scraper.UserHandle
-	LikedByUserHandle     scraper.UserHandle
+	FromUserHandle        scraper.UserHandle   // Tweeted by this user
+	RetweetedByUserHandle scraper.UserHandle   // Retweeted by this user
+	ByUserHandle          scraper.UserHandle   // Either tweeted or retweeted by this user
+	ToUserHandles         []scraper.UserHandle // In reply to these users
+	LikedByUserHandle     scraper.UserHandle   // Liked by this user
+	ListID                scraper.ListID       // Either tweeted or retweeted by users from this List
 	SinceTimestamp        scraper.Timestamp
 	UntilTimestamp        scraper.Timestamp
 	FilterLinks           Filter
@@ -179,6 +180,21 @@ func NewTimelineCursor() Cursor {
 	}
 }
 
+// Generate a cursor appropriate for showing a List feed
+func NewListCursor(list_id scraper.ListID) Cursor {
+	return Cursor{
+		Keywords:       []string{},
+		ToUserHandles:  []scraper.UserHandle{},
+		ListID:         list_id,
+		SinceTimestamp: scraper.TimestampFromUnix(0),
+		UntilTimestamp: scraper.TimestampFromUnix(0),
+		CursorPosition: CURSOR_START,
+		CursorValue:    0,
+		SortOrder:      SORT_ORDER_NEWEST,
+		PageSize:       50,
+	}
+}
+
 // Generate a cursor appropriate for fetching a User Feed
 func NewUserFeedCursor(h scraper.UserHandle) Cursor {
 	return Cursor{
@@ -375,6 +391,10 @@ func (p Profile) NextPage(c Cursor, current_user_id scraper.UserID) (Feed, error
 		where_clauses = append(where_clauses, "by_user_id = (select id from users where handle like ?)")
 		bind_values = append(bind_values, c.ByUserHandle)
 	}
+	if c.ListID != 0 {
+		where_clauses = append(where_clauses, "by_user_id in (select user_id from list_users where list_id = ?)")
+		bind_values = append(bind_values, c.ListID)
+	}
 
 	// Since and until timestamps
 	if c.SinceTimestamp.Unix() != 0 {

sample_data/seed_data.sql

@@ -73,6 +73,8 @@ create index if not exists index_list_users_list_id on list_users (list_id);
 create index if not exists index_list_users_user_id on list_users (user_id);
 insert into lists(rowid, name) values (1, "Offline Follows");
 insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;
+insert into lists(rowid, name) values (2, "Bronze Age");
+insert into list_users(list_id, user_id) select 2, id from users where display_name like "%bronze age%";
 
 
 create table tombstone_types (rowid integer primary key,