From 17423b34c159bd14ad45cac27e185b2d6e482dc5 Mon Sep 17 00:00:00 2001 From: Alessio Date: Thu, 3 Aug 2023 12:43:17 -0300 Subject: [PATCH] Add the 'webserver' internal package with UserFeed and TweetDetail routes and templates --- internal/webserver/middlewares.go | 42 ++++ internal/webserver/response_helpers.go | 72 +++++++ internal/webserver/server.go | 202 ++++++++++++++++++ internal/webserver/server_test.go | 138 ++++++++++++ internal/webserver/static/styles.css | 54 +++++ internal/webserver/tpl/home.tpl | 6 + .../webserver/tpl/includes/author_info.tpl | 11 + internal/webserver/tpl/includes/base.tpl | 20 ++ .../webserver/tpl/includes/single_tweet.tpl | 44 ++++ internal/webserver/tpl/tweet_detail.tpl | 17 ++ internal/webserver/tpl/user_feed.tpl | 26 +++ 11 files changed, 632 insertions(+) create mode 100644 internal/webserver/middlewares.go create mode 100644 internal/webserver/response_helpers.go create mode 100644 internal/webserver/server.go create mode 100644 internal/webserver/server_test.go create mode 100644 internal/webserver/static/styles.css create mode 100644 internal/webserver/tpl/home.tpl create mode 100644 internal/webserver/tpl/includes/author_info.tpl create mode 100644 internal/webserver/tpl/includes/base.tpl create mode 100644 internal/webserver/tpl/includes/single_tweet.tpl create mode 100644 internal/webserver/tpl/tweet_detail.tpl create mode 100644 internal/webserver/tpl/user_feed.tpl diff --git a/internal/webserver/middlewares.go b/internal/webserver/middlewares.go new file mode 100644 index 0000000..1b06fcc --- /dev/null +++ b/internal/webserver/middlewares.go @@ -0,0 +1,42 @@ +package webserver + +import ( + "fmt" + "net/http" + "time" +) + +func secureHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // w.Header().Set("Content-Security-Policy", + // "default-src 'self'; style-src 'self' fonts.googleapis.com; font-src fonts.gstatic.com; img-src 'self' pbs.twimg.com") + w.Header().Set("Referrer-Policy", "same-origin") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("X-Frame-Options", "deny") + // w.Header().Set("X-XSS-Protection", "0") + + next.ServeHTTP(w, r) + }) +} + +func (app *Application) logRequest(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t := time.Now() + next.ServeHTTP(w, r) + duration := time.Since(t) + + app.accessLog.Printf("%s - %s %s %s\t%s", r.RemoteAddr, r.Proto, r.Method, r.URL.RequestURI(), duration) + }) +} + +func (app *Application) recoverPanic(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.Header().Set("Connection", "close") + app.error_500(w, fmt.Errorf("%s", err)) + } + }() + next.ServeHTTP(w, r) + }) +} diff --git a/internal/webserver/response_helpers.go b/internal/webserver/response_helpers.go new file mode 100644 index 0000000..5193e22 --- /dev/null +++ b/internal/webserver/response_helpers.go @@ -0,0 +1,72 @@ +package webserver + +import ( + "bytes" + "fmt" + "net/http" + "runtime/debug" + + "html/template" + "path/filepath" + + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +func panic_if(err error) { + if err != nil { + panic(err) + } +} + +// func (app *Application) error_400(w http.ResponseWriter) { +// http.Error(w, "Bad Request", 400) +// } + +func (app *Application) error_400_with_message(w http.ResponseWriter, msg string) { + http.Error(w, fmt.Sprintf("Bad Request\n\n%s", msg), 400) +} + +func (app *Application) error_404(w http.ResponseWriter) { + http.Error(w, "Not Found", 404) +} + +func (app *Application) error_500(w http.ResponseWriter, err error) { + trace := fmt.Sprintf("%s\n%s", err.Error(), debug.Stack()) + err2 := app.ErrorLog.Output(2, trace) // Magic + if err2 != nil { + panic(err2) + } + http.Error(w, "Server error :(", 500) +} + +// Render the given template using a bytes.Buffer. This avoids the possibility of failing partway +// through the rendering, and sending an imcomplete response with "Bad Request" or "Server Error" at the end. +func (app *Application) buffered_render(w http.ResponseWriter, tpl *template.Template, data interface{}) { + // The template to render is always "base". The choice of which template files to parse into the + // template affects what the contents of "main" (inside of "base") will be + buf := new(bytes.Buffer) + err := tpl.ExecuteTemplate(buf, "base", data) + panic_if(err) + + _, err = buf.WriteTo(w) + panic_if(err) +} + +type TweetCollection interface { + Tweet(id scraper.TweetID) scraper.Tweet + User(id scraper.UserID) scraper.User +} + +// Creates a template from the given template file using all the available partials. +// Calls `app.buffered_render` to render the created template. +func (app *Application) buffered_render_template_for(w http.ResponseWriter, tpl_file string, data TweetCollection) { + partials, err := filepath.Glob(get_filepath("tpl/includes/*.tpl")) + panic_if(err) + + tpl, err := template.New("does this matter at all? lol").Funcs( + template.FuncMap{"tweet": data.Tweet, "user": data.User}, + ).ParseFiles(append(partials, get_filepath(tpl_file))...) + panic_if(err) + + app.buffered_render(w, tpl, data) +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go new file mode 100644 index 0000000..744a5b4 --- /dev/null +++ b/internal/webserver/server.go @@ -0,0 +1,202 @@ +package webserver + +import ( + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "html/template" + "log" + "net/http" + "os" + "path" + "runtime" + "strconv" + "strings" + "time" + + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +type Middleware func(http.Handler) http.Handler + +type Application struct { + accessLog *log.Logger + traceLog *log.Logger + InfoLog *log.Logger + ErrorLog *log.Logger + + Middlewares []Middleware + + Profile persistence.Profile +} + +func NewApp(profile persistence.Profile) Application { + ret := Application{ + accessLog: log.New(os.Stdout, "ACCESS\t", log.Ldate|log.Ltime), + traceLog: log.New(os.Stdout, "TRACE\t", log.Ldate|log.Ltime), + InfoLog: log.New(os.Stdout, "INFO\t", log.Ldate|log.Ltime), + ErrorLog: log.New(os.Stderr, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile), + + Profile: profile, + // formDecoder: form.NewDecoder(), + } + ret.Middlewares = []Middleware{ + secureHeaders, + ret.logRequest, + ret.recoverPanic, + } + return ret +} + +func (app *Application) WithMiddlewares() http.Handler { + var ret http.Handler = app + for i := range app.Middlewares { + ret = app.Middlewares[i](ret) + } + return ret +} + +var this_dir string + +func init() { + _, this_file, _, _ := runtime.Caller(0) // `this_file` is absolute path to this source file + this_dir = path.Dir(this_file) +} + +func get_filepath(s string) string { + return path.Join(this_dir, s) +} + +// Manual router implementation. +// I don't like the weird matching behavior of http.ServeMux, and it's not hard to write by hand. +func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" { + app.Home(w, r) + return + } + + parts := strings.Split(r.URL.Path, "/")[1:] + switch parts[0] { + case "static": + http.StripPrefix("/static", http.FileServer(http.Dir(get_filepath("static")))).ServeHTTP(w, r) + case "tweet": + app.TweetDetail(w, r) + default: + app.UserFeed(w, r) + } +} + +func (app *Application) Run(address string) { + srv := &http.Server{ + Addr: address, + ErrorLog: app.ErrorLog, + Handler: app.WithMiddlewares(), + TLSConfig: &tls.Config{ + CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256}, + }, + IdleTimeout: time.Minute, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + } + + app.InfoLog.Printf("Starting server on %s", address) + err := srv.ListenAndServe() + app.ErrorLog.Fatal(err) +} + +func (app *Application) Home(w http.ResponseWriter, r *http.Request) { + app.traceLog.Printf("'Home' handler (path: %q)", r.URL.Path) + tpl, err := template.ParseFiles( + get_filepath("tpl/includes/base.tpl"), + get_filepath("tpl/home.tpl"), + ) + panic_if(err) + app.buffered_render(w, tpl, nil) +} + +type TweetDetailData struct { + persistence.TweetDetailView + MainTweetID scraper.TweetID +} + +func NewTweetDetailData() TweetDetailData { + return TweetDetailData{ + TweetDetailView: persistence.NewTweetDetailView(), + } +} +func (t TweetDetailData) Tweet(id scraper.TweetID) scraper.Tweet { + return t.Tweets[id] +} +func (t TweetDetailData) User(id scraper.UserID) scraper.User { + return t.Users[id] +} + +func to_json(t interface{}) string { + js, err := json.Marshal(t) + if err != nil { + panic(err) + } + return string(js) +} + +func (app *Application) TweetDetail(w http.ResponseWriter, r *http.Request) { + app.traceLog.Printf("'TweetDetail' handler (path: %q)", r.URL.Path) + _, tail := path.Split(r.URL.Path) + tweet_id, err := strconv.Atoi(tail) + if err != nil { + app.error_400_with_message(w, fmt.Sprintf("Invalid tweet ID: %q", tail)) + return + } + + data := NewTweetDetailData() + data.MainTweetID = scraper.TweetID(tweet_id) + + trove, err := app.Profile.GetTweetDetail(data.MainTweetID) + if err != nil { + if errors.Is(err, persistence.ErrNotInDB) { + app.error_404(w) + return + } else { + panic(err) + } + } + app.InfoLog.Printf(to_json(trove)) + data.TweetDetailView = trove + + app.buffered_render_template_for(w, "tpl/tweet_detail.tpl", data) +} + +type UserProfileData struct { + persistence.Feed + scraper.UserID +} + +func (t UserProfileData) Tweet(id scraper.TweetID) scraper.Tweet { + return t.Tweets[id] +} +func (t UserProfileData) User(id scraper.UserID) scraper.User { + return t.Users[id] +} + +func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { + app.traceLog.Printf("'UserFeed' handler (path: %q)", r.URL.Path) + + _, tail := path.Split(r.URL.Path) + + user, err := app.Profile.GetUserByHandle(scraper.UserHandle(tail)) + if err != nil { + app.error_404(w) + return + } + feed, err := app.Profile.GetUserFeed(user.ID, 50, scraper.TimestampFromUnix(0)) + if err != nil { + panic(err) + } + + data := UserProfileData{Feed: feed, UserID: user.ID} + app.InfoLog.Printf(to_json(data)) + + app.buffered_render_template_for(w, "tpl/user_feed.tpl", data) +} diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go new file mode 100644 index 0000000..7279899 --- /dev/null +++ b/internal/webserver/server_test.go @@ -0,0 +1,138 @@ +package webserver_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/andybalholm/cascadia" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/html" + + "gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver" + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" +) + +type CapturingWriter struct { + Writes [][]byte +} + +func (w *CapturingWriter) Write(p []byte) (int, error) { + w.Writes = append(w.Writes, p) + return len(p), nil +} + +var profile persistence.Profile + +func init() { + var err error + profile, err = persistence.LoadProfile("../../sample_data/profile") + if err != nil { + panic(err) + } +} + +func selector(s string) cascadia.Sel { + ret, err := cascadia.Parse(s) + if err != nil { + panic(err) + } + return ret +} + +func do_request(req *http.Request) *http.Response { + recorder := httptest.NewRecorder() + app := webserver.NewApp(profile) + app.ServeHTTP(recorder, req) + return recorder.Result() +} + +// Homepage +// -------- + +func TestHomepage(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/", nil)) + require.Equal(resp.StatusCode, 200) + + root, err := html.Parse(resp.Body) + require.NoError(err) + title_node := cascadia.Query(root, selector("title")) + assert.Equal(title_node.FirstChild.Data, "Offline Twitter | Home") +} + +// User feed +// --------- + +func TestUserFeed(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/cernovich", nil)) + require.Equal(resp.StatusCode, 200) + + root, err := html.Parse(resp.Body) + require.NoError(err) + title_node := cascadia.Query(root, selector("title")) + assert.Equal(title_node.FirstChild.Data, "Offline Twitter | @Cernovich") + + tweet_nodes := cascadia.QueryAll(root, selector(".tweet")) + assert.Len(tweet_nodes, 7) +} + +func TestUserFeedMissing(t *testing.T) { + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/awefhwefhwejh", nil)) + require.Equal(resp.StatusCode, 404) +} + +// Tweet Detail page +// ----------------- + +func TestTweetDetail(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/tweet/1413773185296650241", nil)) + require.Equal(resp.StatusCode, 200) + + root, err := html.Parse(resp.Body) + require.NoError(err) + tweet_nodes := cascadia.QueryAll(root, selector(".tweet")) + assert.Len(tweet_nodes, 4) +} + +func TestTweetDetailMissing(t *testing.T) { + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/tweet/100089", nil)) + require.Equal(resp.StatusCode, 404) +} + +func TestTweetDetailInvalidNumber(t *testing.T) { + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/tweet/fwjgkj", nil)) + require.Equal(resp.StatusCode, 400) +} + +// Static content +// -------------- + +func TestStaticFile(t *testing.T) { + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/static/styles.css", nil)) + require.Equal(resp.StatusCode, 200) +} + +func TestStaticFileNonexistent(t *testing.T) { + require := require.New(t) + + resp := do_request(httptest.NewRequest("GET", "/static/blehblehblehwfe", nil)) + require.Equal(resp.StatusCode, 404) +} diff --git a/internal/webserver/static/styles.css b/internal/webserver/static/styles.css new file mode 100644 index 0000000..c0fdc70 --- /dev/null +++ b/internal/webserver/static/styles.css @@ -0,0 +1,54 @@ +:root { + --color-twitter-text-gray: #536171; +} + + +body { + margin: 0 30%; +} + +.tweet { + padding: 20px; + + outline-color: lightgray; + outline-style: solid; + outline-width: 1px; + border-radius: 20px; +} + +.quoted-tweet { + padding: 20px; + + outline-color: lightgray; + outline-style: solid; + outline-width: 1px; + border-radius: 20px; +} + +.profile-banner-image { + width: 100%; +} + +.unstyled-link { + text-decoration: none; + color: inherit; + line-height: 0; +} + + +.author-info { + display: flex; + align-items: center; + padding: 0.5em 0; +} + +.name-and-handle { + padding: 0 0.5em; +} + +.display-name { + font-weight: bold; +} +.handle { + color: var(--color-twitter-text-gray); +} diff --git a/internal/webserver/tpl/home.tpl b/internal/webserver/tpl/home.tpl new file mode 100644 index 0000000..190b6d1 --- /dev/null +++ b/internal/webserver/tpl/home.tpl @@ -0,0 +1,6 @@ +{{define "title"}}Home{{end}} + + +{{define "main"}} +

Hello!

+{{end}} diff --git a/internal/webserver/tpl/includes/author_info.tpl b/internal/webserver/tpl/includes/author_info.tpl new file mode 100644 index 0000000..c9bcb2c --- /dev/null +++ b/internal/webserver/tpl/includes/author_info.tpl @@ -0,0 +1,11 @@ +{{define "author-info"}} +
+ + + + +
{{.DisplayName}}
+
@{{.Handle}}
+ +
+{{end}} diff --git a/internal/webserver/tpl/includes/base.tpl b/internal/webserver/tpl/includes/base.tpl new file mode 100644 index 0000000..0721ed6 --- /dev/null +++ b/internal/webserver/tpl/includes/base.tpl @@ -0,0 +1,20 @@ +{{define "base"}} + + + + + Offline Twitter | {{template "title" .}} + + + + + +
+

Uhhhh

+
+
+ {{template "main" .}} +
+ + +{{end}} diff --git a/internal/webserver/tpl/includes/single_tweet.tpl b/internal/webserver/tpl/includes/single_tweet.tpl new file mode 100644 index 0000000..a0e3e61 --- /dev/null +++ b/internal/webserver/tpl/includes/single_tweet.tpl @@ -0,0 +1,44 @@ +{{define "tweet"}} +
+ {{$main_tweet := (tweet .)}} + {{$author := (user $main_tweet.UserID)}} + + {{template "author-info" $author}} +
+ {{$main_tweet.Text}} + + {{range $main_tweet.Images}} + + {{end}} + + {{if $main_tweet.QuotedTweetID}} + {{$quoted_tweet := (tweet $main_tweet.QuotedTweetID)}} + {{$quoted_author := (user $quoted_tweet.UserID)}} + +
+ {{template "author-info" $quoted_author}} +
+

{{$quoted_tweet.Text}}

+ {{range $quoted_tweet.Images}} + + {{end}} +

{{$quoted_tweet.PostedAt}}

+
+
+ + {{end}} + +

{{$main_tweet.PostedAt}}

+
+ +
+ {{$main_tweet.NumQuoteTweets}} QTs + {{$main_tweet.NumReplies}} replies + {{$main_tweet.NumRetweets}} retweets + {{$main_tweet.NumLikes}} likes +
+
+ +
+
+{{end}} diff --git a/internal/webserver/tpl/tweet_detail.tpl b/internal/webserver/tpl/tweet_detail.tpl new file mode 100644 index 0000000..e9698d7 --- /dev/null +++ b/internal/webserver/tpl/tweet_detail.tpl @@ -0,0 +1,17 @@ +{{define "title"}}Tweet{{end}} + + +{{define "main"}} + {{range .ParentIDs}} + {{template "tweet" .}} + {{end}} + {{template "tweet" .MainTweetID}} +
+ + {{range .ReplyChains}} + {{range .}} + {{template "tweet" .}} + {{end}} +
+ {{end}} +{{end}} diff --git a/internal/webserver/tpl/user_feed.tpl b/internal/webserver/tpl/user_feed.tpl new file mode 100644 index 0000000..3fc21e9 --- /dev/null +++ b/internal/webserver/tpl/user_feed.tpl @@ -0,0 +1,26 @@ +{{define "title"}}@{{(user .UserID).Handle}}{{end}} + +{{define "main"}} + {{$user := (user .UserID)}} + + + {{template "author-info" $user}} + +

{{$user.Bio}}

+

{{$user.Location}}

+

{{$user.Website}}

+

{{$user.JoinDate}}

+ +
+ {{$user.FollowersCount}} + followers + is following + {{$user.FollowingCount}} +
+ +
+ + {{range .Items}} + {{template "tweet" .TweetID}} + {{end}} +{{end}}