More powerful static file handling

breezewiki.rkt

@@ -1,17 +1,11 @@
 #lang racket/base
-(require racket/path
+(require web-server/servlet-dispatch
-         racket/runtime-path
-         net/url
-         web-server/servlet-dispatch
-         web-server/dispatchers/filesystem-map
          (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure)
          (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer)
          (prefix-in lift: web-server/dispatchers/dispatch-lift)
          (prefix-in filter: web-server/dispatchers/dispatch-filter)
-         (prefix-in files: web-server/dispatchers/dispatch-files)
          "src/config.rkt"
-         "src/reloadable.rkt"
+         "src/reloadable.rkt")
-         "src/server-utils.rkt")
 
 (define-syntax-rule (require-reloadable filename varname)
   (define varname
@@ -23,14 +17,13 @@
 (require-reloadable "src/page-not-found.rkt" page-not-found)
 (require-reloadable "src/page-proxy.rkt" page-proxy)
 (require-reloadable "src/page-search.rkt" page-search)
+(require-reloadable "src/page-static.rkt" static-dispatcher)
 (require-reloadable "src/page-wiki.rkt" page-wiki)
 
 (when (not (config-true? 'debug))
   (set-reload-poll-interval! #f))
 (reload!)
 
-(define-runtime-path path-static "static")
-
 (serve/launch/wait
  #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f)
  #:port (string->number (config-get 'port))
@@ -41,13 +34,5 @@
     (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make page-category))
     (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make page-wiki))
     (filter:make #rx"^/[a-z-]+/search$" (lift:make page-search))
-    (filter:make #rx"^/static/" (files:make
+    static-dispatcher
-                                 #:url->path
-                                 (lambda (u)
-                                   ((make-url->path path-static)
-                                    (struct-copy url u [path (cdr (url-path u))])))
-                                 #:path->mime-type
-                                 (lambda (u)
-                                   (ext->mime-type (path-get-extension u)))
-                                 #:cache-no-cache (config-true? 'debug) #;"browser applies heuristics if unset"))
     (lift:make page-not-found))))

dist.rkt

@@ -1,26 +1,20 @@
 #lang racket/base
-(require racket/path
+(require web-server/servlet-dispatch
-         racket/runtime-path
-         net/url
-         web-server/servlet-dispatch
-         web-server/dispatchers/filesystem-map
          (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure)
          (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer)
          (prefix-in lift: web-server/dispatchers/dispatch-lift)
          (prefix-in filter: web-server/dispatchers/dispatch-filter)
-         (prefix-in files: web-server/dispatchers/dispatch-files)
          "src/config.rkt"
-         "src/server-utils.rkt")
+         "src/reloadable.rkt")
 
 (require (only-in "src/page-category.rkt" page-category))
 (require (only-in "src/page-home.rkt" page-home))
 (require (only-in "src/page-not-found.rkt" page-not-found))
 (require (only-in "src/page-proxy.rkt" page-proxy))
 (require (only-in "src/page-search.rkt" page-search))
+(require (only-in "src/page-static.rkt" static-dispatcher))
 (require (only-in "src/page-wiki.rkt" page-wiki))
 
-(define-runtime-path path-static "static")
-
 (serve/launch/wait
  #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f)
  #:port (string->number (config-get 'port))
@@ -31,13 +25,5 @@
     (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make page-category))
     (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make page-wiki))
     (filter:make #rx"^/[a-z-]+/search$" (lift:make page-search))
-    (filter:make #rx"^/static/" (files:make
+    static-dispatcher
-                                 #:url->path
-                                 (lambda (u)
-                                   ((make-url->path path-static)
-                                    (struct-copy url u [path (cdr (url-path u))])))
-                                 #:path->mime-type
-                                 (lambda (u)
-                                   (ext->mime-type (path-get-extension u)))
-                                 #:cache-no-cache (config-true? 'debug) #;"browser applies heuristics if unset"))
     (lift:make page-not-found))))

src/config.rkt

@@ -49,6 +49,7 @@
         (printf "note: ~a items loaded from config file~n" (length l)))))))
 
 (when (config-true? 'debug)
+  ; all values here are optimised for maximum prettiness
   (parameterize ([pretty-print-columns 80])
     (display "config: ")
     (pretty-write (hash->list config))))

src/page-static.rkt

@@ -0,0 +1,69 @@
+#lang racket/base
+(require racket/path
+         racket/runtime-path
+         net/url
+         web-server/http/request-structs
+         web-server/servlet-dispatch
+         web-server/dispatchers/filesystem-map
+         (only-in web-server/dispatchers/dispatch next-dispatcher)
+         (prefix-in files: web-server/dispatchers/dispatch-files)
+         "config.rkt")
+
+(provide
+ static-dispatcher)
+
+(module+ test
+  (require rackunit))
+
+(define-runtime-path path-static "../static")
+
+(define hash-ext-mime-type
+  (hash #".css" #"text/css"
+        #".png" #"image/png"
+        #".svg" #"image/svg+xml"
+        #".txt" #"text/plain"))
+
+(define (ext->mime-type ext)
+  (hash-ref hash-ext-mime-type ext))
+(module+ test
+  (check-equal? (ext->mime-type #".png") #"image/png"))
+
+(define (make-path segments)
+  (map (λ (seg) (path/param seg '())) segments))
+(module+ test
+  (check-equal? (make-path '("static" "main.css"))
+                (list (path/param "static" '()) (path/param "main.css" '()))))
+
+(define (path-rewriter p)
+  (cond
+    ; url is ^/static/... ?
+    [(equal? (path/param-path (car p)) "static")
+     ; rewrite to ^/... which will be treated as relative to static/ on the filesystem
+     (cdr p)]
+    ; url is literally ^/robots.txt
+    [(equal? p (make-path '("robots.txt")))
+     ; rewrite to ^/... -- it already is!
+     p]
+    ; not going to use the static file dispatcher
+    [#t (next-dispatcher)]))
+(module+ test
+  (check-equal? (path-rewriter (make-path '("static" "main.css")))
+                (make-path '("main.css")))
+  (check-equal? (path-rewriter (make-path '("static" "robots.txt")))
+                (make-path '("robots.txt")))
+  (check-equal? (path-rewriter (make-path '("robots.txt")))
+                (make-path '("robots.txt"))))
+
+(define (static-dispatcher conn old-req)
+  (define old-uri (request-uri old-req))
+  (define old-path (url-path old-uri))
+  (define new-path (path-rewriter old-path))
+  (define new-uri (struct-copy url old-uri [path new-path]))
+  (define new-req (struct-copy request old-req [uri new-uri]))
+  ((files:make
+    #:url->path (lambda (u)
+                  (println u)
+                  ((make-url->path path-static) u))
+    #:path->mime-type (lambda (u) (ext->mime-type (path-get-extension u)))
+    #:cache-no-cache (config-true? 'debug) #;"browser applies heuristics if unset")
+   conn new-req))

src/server-utils.rkt

@@ -1,16 +0,0 @@
-#lang racket/base
-
-(provide
- ext->mime-type)
-
-(module+ test
-  (require rackunit))
-
-(define hash-ext-mime-type
-  (hash #".css" #"text/css"
-        #".svg" #"image/svg+xml"
-        #".png" #"image/png"))
-(define (ext->mime-type ext)
-  (hash-ref hash-ext-mime-type ext))
-(module+ test
-  (check-equal? (ext->mime-type #".png") #"image/png"))

static/robots.txt

@@ -0,0 +1,3 @@
+User-Agent: *
+Disallow: /*/wiki/*
+Disallow: /proxy