Add the 'webserver' internal package with UserFeed and TweetDetail routes and templates

internal/webserver/middlewares.go

@@ -0,0 +1,42 @@
+package webserver
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func secureHeaders(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// w.Header().Set("Content-Security-Policy",
+		//	"default-src 'self'; style-src 'self' fonts.googleapis.com; font-src fonts.gstatic.com; img-src 'self' pbs.twimg.com")
+		w.Header().Set("Referrer-Policy", "same-origin")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("X-Frame-Options", "deny")
+		// w.Header().Set("X-XSS-Protection", "0")
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (app *Application) logRequest(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t := time.Now()
+		next.ServeHTTP(w, r)
+		duration := time.Since(t)
+
+		app.accessLog.Printf("%s - %s %s %s\t%s", r.RemoteAddr, r.Proto, r.Method, r.URL.RequestURI(), duration)
+	})
+}
+
+func (app *Application) recoverPanic(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				w.Header().Set("Connection", "close")
+				app.error_500(w, fmt.Errorf("%s", err))
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}

internal/webserver/response_helpers.go

@@ -0,0 +1,72 @@
+package webserver
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"runtime/debug"
+
+	"html/template"
+	"path/filepath"
+
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+)
+
+func panic_if(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// func (app *Application) error_400(w http.ResponseWriter) {
+// 	http.Error(w, "Bad Request", 400)
+// }
+
+func (app *Application) error_400_with_message(w http.ResponseWriter, msg string) {
+	http.Error(w, fmt.Sprintf("Bad Request\n\n%s", msg), 400)
+}
+
+func (app *Application) error_404(w http.ResponseWriter) {
+	http.Error(w, "Not Found", 404)
+}
+
+func (app *Application) error_500(w http.ResponseWriter, err error) {
+	trace := fmt.Sprintf("%s\n%s", err.Error(), debug.Stack())
+	err2 := app.ErrorLog.Output(2, trace) // Magic
+	if err2 != nil {
+		panic(err2)
+	}
+	http.Error(w, "Server error :(", 500)
+}
+
+// Render the given template using a bytes.Buffer.  This avoids the possibility of failing partway
+// through the rendering, and sending an imcomplete response with "Bad Request" or "Server Error" at the end.
+func (app *Application) buffered_render(w http.ResponseWriter, tpl *template.Template, data interface{}) {
+	// The template to render is always "base".  The choice of which template files to parse into the
+	// template affects what the contents of "main" (inside of "base") will be
+	buf := new(bytes.Buffer)
+	err := tpl.ExecuteTemplate(buf, "base", data)
+	panic_if(err)
+
+	_, err = buf.WriteTo(w)
+	panic_if(err)
+}
+
+type TweetCollection interface {
+	Tweet(id scraper.TweetID) scraper.Tweet
+	User(id scraper.UserID) scraper.User
+}
+
+// Creates a template from the given template file using all the available partials.
+// Calls `app.buffered_render` to render the created template.
+func (app *Application) buffered_render_template_for(w http.ResponseWriter, tpl_file string, data TweetCollection) {
+	partials, err := filepath.Glob(get_filepath("tpl/includes/*.tpl"))
+	panic_if(err)
+
+	tpl, err := template.New("does this matter at all? lol").Funcs(
+		template.FuncMap{"tweet": data.Tweet, "user": data.User},
+	).ParseFiles(append(partials, get_filepath(tpl_file))...)
+	panic_if(err)
+
+	app.buffered_render(w, tpl, data)
+}

internal/webserver/server.go

@@ -0,0 +1,202 @@
+package webserver
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"html/template"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+)
+
+type Middleware func(http.Handler) http.Handler
+
+type Application struct {
+	accessLog *log.Logger
+	traceLog  *log.Logger
+	InfoLog   *log.Logger
+	ErrorLog  *log.Logger
+
+	Middlewares []Middleware
+
+	Profile persistence.Profile
+}
+
+func NewApp(profile persistence.Profile) Application {
+	ret := Application{
+		accessLog: log.New(os.Stdout, "ACCESS\t", log.Ldate|log.Ltime),
+		traceLog:  log.New(os.Stdout, "TRACE\t", log.Ldate|log.Ltime),
+		InfoLog:   log.New(os.Stdout, "INFO\t", log.Ldate|log.Ltime),
+		ErrorLog:  log.New(os.Stderr, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile),
+
+		Profile: profile,
+		// formDecoder: form.NewDecoder(),
+	}
+	ret.Middlewares = []Middleware{
+		secureHeaders,
+		ret.logRequest,
+		ret.recoverPanic,
+	}
+	return ret
+}
+
+func (app *Application) WithMiddlewares() http.Handler {
+	var ret http.Handler = app
+	for i := range app.Middlewares {
+		ret = app.Middlewares[i](ret)
+	}
+	return ret
+}
+
+var this_dir string
+
+func init() {
+	_, this_file, _, _ := runtime.Caller(0) // `this_file` is absolute path to this source file
+	this_dir = path.Dir(this_file)
+}
+
+func get_filepath(s string) string {
+	return path.Join(this_dir, s)
+}
+
+// Manual router implementation.
+// I don't like the weird matching behavior of http.ServeMux, and it's not hard to write by hand.
+func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/" {
+		app.Home(w, r)
+		return
+	}
+
+	parts := strings.Split(r.URL.Path, "/")[1:]
+	switch parts[0] {
+	case "static":
+		http.StripPrefix("/static", http.FileServer(http.Dir(get_filepath("static")))).ServeHTTP(w, r)
+	case "tweet":
+		app.TweetDetail(w, r)
+	default:
+		app.UserFeed(w, r)
+	}
+}
+
+func (app *Application) Run(address string) {
+	srv := &http.Server{
+		Addr:     address,
+		ErrorLog: app.ErrorLog,
+		Handler:  app.WithMiddlewares(),
+		TLSConfig: &tls.Config{
+			CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256},
+		},
+		IdleTimeout:  time.Minute,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+	}
+
+	app.InfoLog.Printf("Starting server on %s", address)
+	err := srv.ListenAndServe()
+	app.ErrorLog.Fatal(err)
+}
+
+func (app *Application) Home(w http.ResponseWriter, r *http.Request) {
+	app.traceLog.Printf("'Home' handler (path: %q)", r.URL.Path)
+	tpl, err := template.ParseFiles(
+		get_filepath("tpl/includes/base.tpl"),
+		get_filepath("tpl/home.tpl"),
+	)
+	panic_if(err)
+	app.buffered_render(w, tpl, nil)
+}
+
+type TweetDetailData struct {
+	persistence.TweetDetailView
+	MainTweetID scraper.TweetID
+}
+
+func NewTweetDetailData() TweetDetailData {
+	return TweetDetailData{
+		TweetDetailView: persistence.NewTweetDetailView(),
+	}
+}
+func (t TweetDetailData) Tweet(id scraper.TweetID) scraper.Tweet {
+	return t.Tweets[id]
+}
+func (t TweetDetailData) User(id scraper.UserID) scraper.User {
+	return t.Users[id]
+}
+
+func to_json(t interface{}) string {
+	js, err := json.Marshal(t)
+	if err != nil {
+		panic(err)
+	}
+	return string(js)
+}
+
+func (app *Application) TweetDetail(w http.ResponseWriter, r *http.Request) {
+	app.traceLog.Printf("'TweetDetail' handler (path: %q)", r.URL.Path)
+	_, tail := path.Split(r.URL.Path)
+	tweet_id, err := strconv.Atoi(tail)
+	if err != nil {
+		app.error_400_with_message(w, fmt.Sprintf("Invalid tweet ID: %q", tail))
+		return
+	}
+
+	data := NewTweetDetailData()
+	data.MainTweetID = scraper.TweetID(tweet_id)
+
+	trove, err := app.Profile.GetTweetDetail(data.MainTweetID)
+	if err != nil {
+		if errors.Is(err, persistence.ErrNotInDB) {
+			app.error_404(w)
+			return
+		} else {
+			panic(err)
+		}
+	}
+	app.InfoLog.Printf(to_json(trove))
+	data.TweetDetailView = trove
+
+	app.buffered_render_template_for(w, "tpl/tweet_detail.tpl", data)
+}
+
+type UserProfileData struct {
+	persistence.Feed
+	scraper.UserID
+}
+
+func (t UserProfileData) Tweet(id scraper.TweetID) scraper.Tweet {
+	return t.Tweets[id]
+}
+func (t UserProfileData) User(id scraper.UserID) scraper.User {
+	return t.Users[id]
+}
+
+func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
+	app.traceLog.Printf("'UserFeed' handler (path: %q)", r.URL.Path)
+
+	_, tail := path.Split(r.URL.Path)
+
+	user, err := app.Profile.GetUserByHandle(scraper.UserHandle(tail))
+	if err != nil {
+		app.error_404(w)
+		return
+	}
+	feed, err := app.Profile.GetUserFeed(user.ID, 50, scraper.TimestampFromUnix(0))
+	if err != nil {
+		panic(err)
+	}
+
+	data := UserProfileData{Feed: feed, UserID: user.ID}
+	app.InfoLog.Printf(to_json(data))
+
+	app.buffered_render_template_for(w, "tpl/user_feed.tpl", data)
+}

internal/webserver/server_test.go

@@ -0,0 +1,138 @@
+package webserver_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/andybalholm/cascadia"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/html"
+
+	"gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver"
+	"gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence"
+)
+
+type CapturingWriter struct {
+	Writes [][]byte
+}
+
+func (w *CapturingWriter) Write(p []byte) (int, error) {
+	w.Writes = append(w.Writes, p)
+	return len(p), nil
+}
+
+var profile persistence.Profile
+
+func init() {
+	var err error
+	profile, err = persistence.LoadProfile("../../sample_data/profile")
+	if err != nil {
+		panic(err)
+	}
+}
+
+func selector(s string) cascadia.Sel {
+	ret, err := cascadia.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
+func do_request(req *http.Request) *http.Response {
+	recorder := httptest.NewRecorder()
+	app := webserver.NewApp(profile)
+	app.ServeHTTP(recorder, req)
+	return recorder.Result()
+}
+
+// Homepage
+// --------
+
+func TestHomepage(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/", nil))
+	require.Equal(resp.StatusCode, 200)
+
+	root, err := html.Parse(resp.Body)
+	require.NoError(err)
+	title_node := cascadia.Query(root, selector("title"))
+	assert.Equal(title_node.FirstChild.Data, "Offline Twitter | Home")
+}
+
+// User feed
+// ---------
+
+func TestUserFeed(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/cernovich", nil))
+	require.Equal(resp.StatusCode, 200)
+
+	root, err := html.Parse(resp.Body)
+	require.NoError(err)
+	title_node := cascadia.Query(root, selector("title"))
+	assert.Equal(title_node.FirstChild.Data, "Offline Twitter | @Cernovich")
+
+	tweet_nodes := cascadia.QueryAll(root, selector(".tweet"))
+	assert.Len(tweet_nodes, 7)
+}
+
+func TestUserFeedMissing(t *testing.T) {
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/awefhwefhwejh", nil))
+	require.Equal(resp.StatusCode, 404)
+}
+
+// Tweet Detail page
+// -----------------
+
+func TestTweetDetail(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/tweet/1413773185296650241", nil))
+	require.Equal(resp.StatusCode, 200)
+
+	root, err := html.Parse(resp.Body)
+	require.NoError(err)
+	tweet_nodes := cascadia.QueryAll(root, selector(".tweet"))
+	assert.Len(tweet_nodes, 4)
+}
+
+func TestTweetDetailMissing(t *testing.T) {
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/tweet/100089", nil))
+	require.Equal(resp.StatusCode, 404)
+}
+
+func TestTweetDetailInvalidNumber(t *testing.T) {
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/tweet/fwjgkj", nil))
+	require.Equal(resp.StatusCode, 400)
+}
+
+// Static content
+// --------------
+
+func TestStaticFile(t *testing.T) {
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/static/styles.css", nil))
+	require.Equal(resp.StatusCode, 200)
+}
+
+func TestStaticFileNonexistent(t *testing.T) {
+	require := require.New(t)
+
+	resp := do_request(httptest.NewRequest("GET", "/static/blehblehblehwfe", nil))
+	require.Equal(resp.StatusCode, 404)
+}

internal/webserver/static/styles.css

@@ -0,0 +1,54 @@
+:root {
+	--color-twitter-text-gray: #536171;
+}
+
+
+body {
+	margin: 0 30%;
+}
+
+.tweet {
+	padding: 20px;
+
+	outline-color: lightgray;
+	outline-style: solid;
+	outline-width: 1px;
+	border-radius: 20px;
+}
+
+.quoted-tweet {
+	padding: 20px;
+
+	outline-color: lightgray;
+	outline-style: solid;
+	outline-width: 1px;
+	border-radius: 20px;
+}
+
+.profile-banner-image {
+	width: 100%;
+}
+
+.unstyled-link {
+	text-decoration: none;
+	color: inherit;
+	line-height: 0;
+}
+
+
+.author-info {
+	display: flex;
+	align-items: center;
+	padding: 0.5em 0;
+}
+
+.name-and-handle {
+	padding: 0 0.5em;
+}
+
+.display-name {
+	font-weight: bold;
+}
+.handle {
+	color: var(--color-twitter-text-gray);
+}

internal/webserver/tpl/home.tpl

@@ -0,0 +1,6 @@
+{{define "title"}}Home{{end}}
+
+
+{{define "main"}}
+  <h1>Hello!</h1>
+{{end}}

internal/webserver/tpl/includes/author_info.tpl

@@ -0,0 +1,11 @@
+{{define "author-info"}}
+<div class="author-info">
+  <a class="unstyled-link" href="/{{.Handle}}">
+    <img style="border-radius: 50%; width: 50px; display: inline;" src="{{.ProfileImageUrl}}" />
+  </a>
+  <span class="name-and-handle">
+    <div class="display-name">{{.DisplayName}}</div>
+    <div class="handle">@{{.Handle}}</div>
+  </span>
+</div>
+{{end}}

internal/webserver/tpl/includes/base.tpl

@@ -0,0 +1,20 @@
+{{define "base"}}
+  <!doctype html>
+  <html lang='en'>
+    <head>
+      <meta charset='utf-8'>
+      <title>Offline Twitter | {{template "title" .}}</title>
+      <link rel='stylesheet' href='/static/styles.css'>
+      <link rel='shortcut icon' href='/static/img/favicon.ico' type='image/x-icon'>
+      <!-- <link rel='stylesheet' href='https://fonts.googleapis.com/css?family=Ubuntu+Mono:400,700'> -->
+    </head>
+    <body>
+      <header>
+        <h1><a href='/'>Uhhhh</a></h1>
+      </header>
+      <main>
+        {{template "main" .}}
+      </main>
+    </body>
+  </html>
+{{end}}

internal/webserver/tpl/includes/single_tweet.tpl

@@ -0,0 +1,44 @@
+{{define "tweet"}}
+<div class="tweet">
+  {{$main_tweet := (tweet .)}}
+  {{$author := (user $main_tweet.UserID)}}
+
+  {{template "author-info" $author}}
+  <div class="tweet-content">
+    <a href="/tweet/{{$main_tweet.ID}}" style="color: inherit; text-decoration: none" >{{$main_tweet.Text}}</a>
+
+    {{range $main_tweet.Images}}
+      <img src="{{.RemoteURL}}" style="max-width: 45%"/>
+    {{end}}
+
+    {{if $main_tweet.QuotedTweetID}}
+      {{$quoted_tweet := (tweet $main_tweet.QuotedTweetID)}}
+      {{$quoted_author := (user $quoted_tweet.UserID)}}
+      <a href="/tweet/{{$quoted_tweet.ID}}">
+        <div class="quoted-tweet" style="padding: 20px; outline-color: lightgray; outline-style: solid; outline-width: 1px; border-radius: 20px">
+          {{template "author-info" $quoted_author}}
+          <div class="quoted-tweet-content">
+            <p>{{$quoted_tweet.Text}}</p>
+            {{range $quoted_tweet.Images}}
+              <img src="{{.RemoteURL}}" style="max-width: 45%"/>
+            {{end}}
+            <p>{{$quoted_tweet.PostedAt}}</p>
+          </div>
+        </div>
+      </a>
+    {{end}}
+
+    <p>{{$main_tweet.PostedAt}}</p>
+  </div>
+
+  <div class="interactions-bar">
+    <span>{{$main_tweet.NumQuoteTweets}} QTs</span>
+    <span>{{$main_tweet.NumReplies}} replies</span>
+    <span>{{$main_tweet.NumRetweets}} retweets</span>
+    <span>{{$main_tweet.NumLikes}} likes</span>
+  </div>
+  <div class="interaction-buttons">
+
+  </div>
+</div>
+{{end}}

internal/webserver/tpl/tweet_detail.tpl

@@ -0,0 +1,17 @@
+{{define "title"}}Tweet{{end}}
+
+
+{{define "main"}}
+  {{range .ParentIDs}}
+    {{template "tweet" .}}
+  {{end}}
+  {{template "tweet" .MainTweetID}}
+  <hr />
+
+  {{range .ReplyChains}}
+    {{range .}}
+      {{template "tweet" .}}
+    {{end}}
+    <hr />
+  {{end}}
+{{end}}

internal/webserver/tpl/user_feed.tpl

@@ -0,0 +1,26 @@
+{{define "title"}}@{{(user .UserID).Handle}}{{end}}
+
+{{define "main"}}
+  {{$user := (user .UserID)}}
+  <img class="profile-banner-image" src="{{$user.BannerImageUrl}}" />
+
+  {{template "author-info" $user}}
+  <button>{{if $user.IsFollowed}}Unfollow{{else}}Follow{{end}}</button>
+  <p class="user-bio">{{$user.Bio}}</p>
+  <p class="user-location">{{$user.Location}}</p>
+  <p class="user-website">{{$user.Website}}</p>
+  <p class="user-join-date">{{$user.JoinDate}}</p>
+
+  <div class="followers-followees-container">
+    <span class="followers-count">{{$user.FollowersCount}}</span>
+    <span class="followers-label">followers</span>
+    <span class="following-label">is following</span>
+    <span class="following-count">{{$user.FollowingCount}}</span>
+  </div>
+
+  <hr/>
+
+  {{range .Items}}
+    {{template "tweet" .TweetID}}
+  {{end}}
+{{end}}