From 10cdd260e0a6bec81c8be0a261851661d5a30059 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:00:33 +1300 Subject: [PATCH 1/5] Close response after error in proxy --- src/page-proxy.rkt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/page-proxy.rkt b/src/page-proxy.rkt index f494e28..8dbf0f9 100644 --- a/src/page-proxy.rkt +++ b/src/page-proxy.rkt @@ -18,13 +18,16 @@ (match (dict-ref (url-query (request-uri req)) 'dest #f) [(? string? dest) (if (is-fandom-url? dest) - (response-handler + (response-handler ; catches and reports errors (let ([dest-r (easy:get dest #:stream? #t)]) - (response/output - #:code (easy:response-status-code dest-r) - #:mime-type (easy:response-headers-ref dest-r 'content-type) - (λ (out) - (copy-port (easy:response-output dest-r) out) - (easy:response-close! dest-r))))) + (with-handlers ([exn:fail? (λ (e) ; cleans up and re-throws + (easy:response-close! dest-r) + (raise e))]) + (response/output + #:code (easy:response-status-code dest-r) + #:mime-type (easy:response-headers-ref dest-r 'content-type) + (λ (out) + (copy-port (easy:response-output dest-r) out) + (easy:response-close! dest-r)))))) (next-dispatcher))] [#f (next-dispatcher)])) From ece762fc5b9b4632674cc1b86c50a3a1ecca4423 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:13:07 +1300 Subject: [PATCH 2/5] Unify "out: " logging to a function --- src/config.rkt | 1 + src/data.rkt | 2 +- src/page-category.rkt | 4 ++-- src/page-search.rkt | 2 +- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 10 +++++++++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index bdf4440..4c8fca9 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -26,6 +26,7 @@ (canonical_origin . "") (debug . "false") (instance_is_official . "false") ; please don't turn this on, or you will make me very upset + (log_outgoing . "true") (port . "10416") (strict_proxy . "true"))) diff --git a/src/data.rkt b/src/data.rkt index f7a0d1f..8eb0cd6 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -20,7 +20,7 @@ ("siprop" . "rightsinfo") ("format" . "json") ("formatversion" . "2"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define res (easy:get dest-url)) (define data (easy:response-json res)) (license (jp "/query/rightsinfo/text" data) diff --git a/src/page-category.rkt b/src/page-category.rkt index f9eb974..f7c43b2 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -75,7 +75,7 @@ ("cmlimit" . "max") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] [page-data (define dest-url @@ -86,7 +86,7 @@ ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] [license (license-auto wikiname)]) diff --git a/src/page-search.rkt b/src/page-search.rkt index 61b1212..387deab 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -68,7 +68,7 @@ ("format" . "json"))))) (thread-let - ([dest-res (printf "out: ~a~n" dest-url) + ([dest-res (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] [license (license-auto wikiname)]) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 96410d0..461594a 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -246,7 +246,7 @@ ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] [license (license-auto wikiname)]) diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 934a228..20b9b42 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -1,6 +1,7 @@ #lang typed/racket/base (require racket/string "pure-utils.rkt") +(require/typed "config.rkt" [config-true? (Symbol -> Boolean)]) (provide ; make a query string from an association list of strings @@ -8,7 +9,9 @@ ; make a proxied version of a fandom url u-proxy-url ; check whether a url is on a domain controlled by fandom - is-fandom-url?) + is-fandom-url? + ; prints "out: " + log-outgoing) (module+ test (require "typed-rackunit.rkt")) @@ -69,3 +72,8 @@ is-fandom-url? (λ ([v : String]) (string-append "/proxy?" (params->query `(("dest" . ,url))))) url)) + +(: log-outgoing (String -> Void)) +(define (log-outgoing url-string) + (when (config-true? 'log_outgoing) + (printf "out: ~a~n" url-string))) From 57e700cef547da41e95884a33ca2153a8f48f461 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 2 Oct 2022 16:44:44 +0700 Subject: [PATCH 3/5] Deduplicate the wiki name regex --- src/dispatcher-tree.rkt | 11 ++++++----- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 6 +++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 93532ef..1a43458 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -8,7 +8,8 @@ (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) (prefix-in lift: web-server/dispatchers/dispatch-lift) (prefix-in filter: web-server/dispatchers/dispatch-filter) - "config.rkt") + "config.rkt" + "url-utils.rkt") (provide ; syntax to make the hashmap from names @@ -43,9 +44,9 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/search$" (lift:make (hash-ref ds 'page-search))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}(/(wiki(/)?)?)?$" (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/search$" wikiname-regex)) (lift:make (hash-ref ds 'page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" wikiname-regex)) (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 461594a..4d6fc18 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -143,7 +143,7 @@ (λ (href) ((compose1 (λ (href) (regexp-replace #rx"^(/wiki/.*)" href (format "/~a\\1" wikiname))) - (λ (href) (regexp-replace #px"^https://([a-zA-Z0-9-]{3,50}).fandom.com(/wiki/.*)" href "/\\1\\2"))) + (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" wikiname-regex)) href "/\\1\\2"))) href))) ; add noreferrer to a.image (curry u diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 20b9b42..07133e7 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -4,6 +4,8 @@ (require/typed "config.rkt" [config-true? (Symbol -> Boolean)]) (provide + ; regex to match wiki names + wikiname-regex ; make a query string from an association list of strings params->query ; make a proxied version of a fandom url @@ -16,6 +18,8 @@ (module+ test (require "typed-rackunit.rkt")) +(define wikiname-regex "[a-zA-Z0-9-]{3,50}") + ;; https://url.spec.whatwg.org/#urlencoded-serializing (define urlencoded-set '(#\! #\' #\( #\) #\~ ; urlencoded set @@ -60,7 +64,7 @@ (: is-fandom-url? (String -> Boolean)) (define (is-fandom-url? url) - (regexp-match? #px"^https://static.wikia.nocookie.net/|^https://[a-zA-Z0-9-]{3,50}.fandom.com/" url)) + (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" wikiname-regex)) url)) (module+ test (check-true (is-fandom-url? "https://static.wikia.nocookie.net/wikiname/images/2/2f/SomeImage.jpg/revision/latest?cb=20110210094136")) (check-true (is-fandom-url? "https://test.fandom.com/wiki/Some_Page")) From 79f04565c72acc683b4e6a153d5623adb2bf25da Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:18:30 +1300 Subject: [PATCH 4/5] Rename wikiname-regex to px-wikiname I find prefix notation more natural to represent the type/kind of the thing. --- src/dispatcher-tree.rkt | 8 ++++---- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 1a43458..f2c1412 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -44,9 +44,9 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (filter:make (pregexp (format "^/~a/wiki/Category:.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-category))) - (filter:make (pregexp (format "^/~a/wiki/.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-wiki))) - (filter:make (pregexp (format "^/~a/search$" wikiname-regex)) (lift:make (hash-ref ds 'page-search))) - (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" wikiname-regex)) (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 4d6fc18..a218dfe 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -143,7 +143,7 @@ (λ (href) ((compose1 (λ (href) (regexp-replace #rx"^(/wiki/.*)" href (format "/~a\\1" wikiname))) - (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" wikiname-regex)) href "/\\1\\2"))) + (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" px-wikiname)) href "/\\1\\2"))) href))) ; add noreferrer to a.image (curry u diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 07133e7..a55a208 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -5,7 +5,7 @@ (provide ; regex to match wiki names - wikiname-regex + px-wikiname ; make a query string from an association list of strings params->query ; make a proxied version of a fandom url @@ -18,7 +18,7 @@ (module+ test (require "typed-rackunit.rkt")) -(define wikiname-regex "[a-zA-Z0-9-]{3,50}") +(define px-wikiname "[a-zA-Z0-9-]{3,50}") ;; https://url.spec.whatwg.org/#urlencoded-serializing @@ -64,7 +64,7 @@ (: is-fandom-url? (String -> Boolean)) (define (is-fandom-url? url) - (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" wikiname-regex)) url)) + (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" px-wikiname)) url)) (module+ test (check-true (is-fandom-url? "https://static.wikia.nocookie.net/wikiname/images/2/2f/SomeImage.jpg/revision/latest?cb=20110210094136")) (check-true (is-fandom-url? "https://test.fandom.com/wiki/Some_Page")) From 6b176e3f8f58152620e0a18c7fa57e3fb229a8bb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 22:00:44 +1300 Subject: [PATCH 5/5] Migrate config.rkt to Typed Racket --- src/config.rkt | 44 +++++++++++++++++++++++++++++++++----------- src/url-utils.rkt | 2 +- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index 4c8fca9..6fc029f 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -1,23 +1,32 @@ -#lang racket/base +#lang typed/racket/base (require racket/function racket/pretty racket/runtime-path - racket/string - ini) + racket/string) +(require/typed ini + [#:opaque Ini ini?] + [read-ini (Input-Port -> Ini)] + [ini->hash (Ini -> (Immutable-HashTable Symbol (Immutable-HashTable Symbol String)))]) (provide config-parameter config-true? config-get) +(module+ test + (require "typed-rackunit.rkt")) + (define-runtime-path path-config "../config.ini") +(: config-parameter (Symbol -> (Parameterof String))) (define (config-parameter key) (hash-ref config key)) +(: config-true? (Symbol -> Boolean)) (define (config-true? key) (not (member ((config-parameter key)) '("" "false")))) +(: config-get (Symbol -> String)) (define (config-get key) ((config-parameter key))) @@ -56,18 +65,24 @@ (define env-alist (let ([e-names (environment-variables-names (current-environment-variables))] - [e-ref (λ (name) (bytes->string/latin-1 (environment-variables-ref (current-environment-variables) name)))]) - (map (λ (name) (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) - (e-ref name))) - (filter (λ (name) (string-prefix? (string-downcase (bytes->string/latin-1 name)) "bw_")) e-names)))) + [e-ref (λ ([name : Bytes]) + (bytes->string/latin-1 + (cast (environment-variables-ref (current-environment-variables) name) + Bytes)))]) + (map (λ ([name : Bytes]) + (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) + (e-ref name))) + (filter (λ ([name : Bytes]) (string-prefix? (string-downcase (bytes->string/latin-1 name)) + "bw_")) + e-names)))) (when (> (length env-alist) 0) (printf "note: ~a items loaded from environment variables~n" (length env-alist))) (define combined-alist (append default-config loaded-alist env-alist)) (define config - (make-hasheq - (map (λ (pair) + (make-immutable-hasheq + (map (λ ([pair : (Pairof Symbol String)]) (cons (car pair) (make-parameter (cdr pair)))) combined-alist))) @@ -75,8 +90,8 @@ ; all values here are optimised for maximum prettiness (parameterize ([pretty-print-columns 80]) (display "config: ") - (pretty-write (sort - (hash->list (make-hasheq combined-alist)) + (pretty-write ((inst sort (Pairof Symbol String)) + (hash->list (make-immutable-hasheq combined-alist)) symbol Boolean)]) (provide ; regex to match wiki names