From cc138a07aa4f01464d0894c138feeedf071f9421 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 4 Sep 2022 13:38:30 +1200 Subject: [PATCH 001/166] Rename config options to use underscores Makes more sense for the INI. --- src/application-globals.rkt | 12 ++++++------ src/config.rkt | 7 ++++--- src/page-category.rkt | 2 +- src/page-home.rkt | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 49ea4366..f7d9d67d 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -26,7 +26,7 @@ `(html (head (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) - (title ,(format "~a | ~a" title (config-get 'application-name))) + (title ,(format "~a | ~a" title (config-get 'application_name))) (style ":root { --theme-page-background-color: #dfdfe0 }") ; fallback in case styles don't load fast enough ,@(map (λ (url) `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) @@ -53,22 +53,22 @@ (img (@ (class "my-logo") (src "/static/breezewiki.svg")))) (p (a (@ (href "https://gitdab.com/cadence/breezewiki")) - ,(format "~a source code" (config-get 'application-name)))) + ,(format "~a source code" (config-get 'application_name)))) (p (a (@ (href "https://lists.sr.ht/~cadence/breezewiki-discuss")) "Discussions / Bug reports / Feature requests")) - ,(if (config-get 'instance-is-official) - `(p ,(format "This instance is run by the ~a developer, " (config-get 'application-name)) + ,(if (config-get 'instance_is_official) + `(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) (a (@ (href "https://cadence.moe/contact")) "Cadence.")) `(p - ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application-name))))) + ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name))))) (div (p "This page displays proxied content from " (a (@ (href ,source-url) (rel "noreferrer")) ,source-url) ". Text content is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " (a (@ (href "https://www.fandom.com/licensing")) "see license info.") " Media files may have different copying restrictions.") - (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application-name)))))))))))) + (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))))))))))) (module+ test (check-not-false (xexp->html (generate-wiki-page "" "test" "test" '(template))))) diff --git a/src/config.rkt b/src/config.rkt index 8e785ec0..38433f95 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -16,10 +16,11 @@ (hash-ref config key)) (define default-config - '((port . "10416") + '((application_name . "BreezeWiki") + (canonical_origin . "") (debug . "false") - (instance-is-official . "false") ; please don't turn this on, or you will make me very upset - (application-name . "BreezeWiki"))) + (instance_is_official . "false") ; please don't turn this on, or you will make me very upset + (port . "10416"))) (define config (make-hasheq diff --git a/src/page-category.rkt b/src/page-category.rkt index 83f0f030..bfc689fa 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -59,7 +59,7 @@ (define data (easy:response-json dest-res)) (define body (generate-results-page dest-url wikiname prefixed-category data)) - (when (config-get 'debug) + (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid (xexp->html body)) diff --git a/src/page-home.rkt b/src/page-home.rkt index 4a87a1b1..102ab687 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -56,20 +56,20 @@ (footer (@ (class "custom-footer")) (div (@ (class "internal-footer")) (img (@ (class "my-logo") (src "/static/breezewiki.svg"))) - ,(if (config-get 'instance-is-official) + ,(if (config-get 'instance_is_official) `(div - (p ,(format "This instance is run by the ~a developer, " (config-get 'application-name)) + (p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) (a (@ (href "https://cadence.moe/contact")) "Cadence.")) (p "Hosting generously provided by " (a (@ (href "http://alphamethyl.barr0w.net/")) "alphamethyl."))) `(p - ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application-name)))) + ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name)))) (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " (a (@ (href "https://www.fandom.com/licensing")) "see license info.") " Media files and official Fandom documents have different copying restrictions.") - (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application-name))))))))))) + (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name))))))))))) (module+ test (check-not-false (xexp->html body))) From 20a40438892ea9d4fdbf72c3fa92142cc91f639a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 4 Sep 2022 22:13:36 +1200 Subject: [PATCH 002/166] Simplify files even more - breezewiki.rkt now always sets up the hot-reload watcher - dispatcher logic moved to dispatcher-tree.rkt - dispatcher-tree is a macro that doesn't care about the order of its forms --- breezewiki.rkt | 24 ++++++++++-------------- dist.rkt | 23 ++++++++++------------- src/dispatcher-tree.rkt | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 src/dispatcher-tree.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index 0421e122..ac60e1f5 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -1,10 +1,7 @@ #lang racket/base (require web-server/servlet-dispatch - (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure) - (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) - (prefix-in lift: web-server/dispatchers/dispatch-lift) - (prefix-in filter: web-server/dispatchers/dispatch-filter) "src/config.rkt" + "src/dispatcher-tree.rkt" "src/reloadable.rkt") (define-syntax-rule (require-reloadable filename varname) @@ -20,8 +17,6 @@ (require-reloadable "src/page-static.rkt" static-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) -(when (not (config-true? 'debug)) - (set-reload-poll-interval! #f)) (reload!) (define ch (make-channel)) @@ -31,13 +26,14 @@ #:port (string->number (config-get 'port)) (λ (quit) (channel-put ch (lambda () (semaphore-post quit))) - (sequencer:make - (pathprocedure:make "/" page-home) - (pathprocedure:make "/proxy" page-proxy) - (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make page-category)) - (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make page-wiki)) - (filter:make #rx"^/[a-z-]+/search$" (lift:make page-search)) - static-dispatcher - (lift:make page-not-found))))) + (dispatcher-tree + ; order of these does not matter + page-category + page-home + page-not-found + page-proxy + page-search + page-wiki + static-dispatcher)))) (define server-t (thread start)) (define quit (channel-get ch)) diff --git a/dist.rkt b/dist.rkt index 9e190efd..289f9bd3 100644 --- a/dist.rkt +++ b/dist.rkt @@ -1,11 +1,7 @@ #lang racket/base (require web-server/servlet-dispatch - (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure) - (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) - (prefix-in lift: web-server/dispatchers/dispatch-lift) - (prefix-in filter: web-server/dispatchers/dispatch-filter) "src/config.rkt" - "src/reloadable.rkt") + "src/dispatcher-tree.rkt") (require (only-in "src/page-category.rkt" page-category)) (require (only-in "src/page-home.rkt" page-home)) @@ -19,11 +15,12 @@ #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f) #:port (string->number (config-get 'port)) (λ (quit) - (sequencer:make - (pathprocedure:make "/" page-home) - (pathprocedure:make "/proxy" page-proxy) - (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make page-category)) - (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make page-wiki)) - (filter:make #rx"^/[a-z-]+/search$" (lift:make page-search)) - static-dispatcher - (lift:make page-not-found)))) + (dispatcher-tree + ; order of these does not matter + page-category + page-home + page-not-found + page-proxy + page-search + page-wiki + static-dispatcher))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt new file mode 100644 index 00000000..626ee093 --- /dev/null +++ b/src/dispatcher-tree.rkt @@ -0,0 +1,36 @@ +#lang racket/base +(require (for-syntax racket/base) + (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure) + (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) + (prefix-in lift: web-server/dispatchers/dispatch-lift) + (prefix-in filter: web-server/dispatchers/dispatch-filter)) + +(provide + ; syntax to make the hashmap from names + dispatcher-tree + ; procedure to make the tree from the hashmap + make-dispatcher-tree) + +; make a hashmap out of the provided names and call make-dispatcher-tree with it +(define-syntax (dispatcher-tree stx) + ; the arguments, which are names of dispatcher variables + (define names (cdr (syntax->list stx))) + ; map each name to syntax of a '(name . ,name) + (define alist (map (λ (xe) ; xe is the syntax of a name + ; return instead syntax of a cons cell + (datum->syntax stx `(cons ',xe ,xe))) + names)) + ; make syntax to make the hash + (define ds (datum->syntax stx `(make-hasheq (list ,@alist)))) + ; don't forget that I'm returning *code* - return a call to the function + (datum->syntax stx `(make-dispatcher-tree ,ds))) + +(define (make-dispatcher-tree ds) + (sequencer:make + (pathprocedure:make "/" (hash-ref ds 'page-home)) + (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) + (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) + (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) + (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) + (hash-ref ds 'static-dispatcher) + (lift:make (hash-ref ds 'page-not-found)))) From 8a720031704298240d23b99ee44691100ae6d590 Mon Sep 17 00:00:00 2001 From: bopol Date: Sat, 3 Sep 2022 21:22:20 +0200 Subject: [PATCH 003/166] Only display footer in column if space's available Otherwise it would cause an overflow e.g. on mobile --- static/main.css | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/static/main.css b/static/main.css index 33d72649..b06c86e3 100644 --- a/static/main.css +++ b/static/main.css @@ -62,7 +62,11 @@ p { } .custom-footer__cols { display: grid; - grid-template-columns: 1fr 1fr; +} +@media (min-width: 560px) { + .custom-footer__cols { + grid-template-columns: 1fr 1fr; + } } .my-logo { margin-bottom: 8px; From 4815db4063949348f48157c00f0f276879c8ab44 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 01:32:45 +1200 Subject: [PATCH 004/166] Dispatch based on Host header (subdomain support) --- breezewiki.rkt | 4 +++- dist.rkt | 4 +++- src/dispatcher-tree.rkt | 53 ++++++++++++++++++++++++++++++++++------- src/page-home.rkt | 2 +- src/page-subdomain.rkt | 36 ++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 src/page-subdomain.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index ac60e1f5..dbca4285 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -15,6 +15,7 @@ (require-reloadable "src/page-proxy.rkt" page-proxy) (require-reloadable "src/page-search.rkt" page-search) (require-reloadable "src/page-static.rkt" static-dispatcher) +(require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) (reload!) @@ -34,6 +35,7 @@ page-proxy page-search page-wiki - static-dispatcher)))) + static-dispatcher + subdomain-dispatcher)))) (define server-t (thread start)) (define quit (channel-get ch)) diff --git a/dist.rkt b/dist.rkt index 289f9bd3..d0720be1 100644 --- a/dist.rkt +++ b/dist.rkt @@ -9,6 +9,7 @@ (require (only-in "src/page-proxy.rkt" page-proxy)) (require (only-in "src/page-search.rkt" page-search)) (require (only-in "src/page-static.rkt" static-dispatcher)) +(require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) (require (only-in "src/page-wiki.rkt" page-wiki)) (serve/launch/wait @@ -23,4 +24,5 @@ page-proxy page-search page-wiki - static-dispatcher))) + static-dispatcher + subdomain-dispatcher))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 626ee093..49306d8a 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -1,9 +1,13 @@ #lang racket/base (require (for-syntax racket/base) + racket/string + net/url + (prefix-in host: web-server/dispatchers/dispatch-host) (prefix-in pathprocedure: web-server/dispatchers/dispatch-pathprocedure) (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) (prefix-in lift: web-server/dispatchers/dispatch-lift) - (prefix-in filter: web-server/dispatchers/dispatch-filter)) + (prefix-in filter: web-server/dispatchers/dispatch-filter) + "config.rkt") (provide ; syntax to make the hashmap from names @@ -11,6 +15,29 @@ ; procedure to make the tree from the hashmap make-dispatcher-tree) +(define-syntax (if/out stx) + (define tree (cdr (syntax->datum stx))) ; condition true false + (define else (cddr tree)) ; the else branch cons cell + (define result + (let walk ([node tree]) + (cond + ; normally, node should be a full cons cell (a pair) but it might be something else. + ; situation: reached the end of a list, empty cons cell + [(null? node) node] + ; situation: reached the end of a list, cons cdr was non-list + [(symbol? node) node] + ; normal situation, full cons cell + ; -- don't go replacing through nested if/out + [(and (pair? node) (eq? 'if/out (car node))) node] + ; -- replace if/in + [(and (pair? node) (eq? 'if/in (car node))) + (append '(if) (cdr node) else)] + ; recurse down pair head and tail + [(pair? node) (cons (walk (car node)) (walk (cdr node)))] + ; something else that can't be recursed into, so pass it through + [#t node]))) + (datum->syntax stx (cons 'if result))) + ; make a hashmap out of the provided names and call make-dispatcher-tree with it (define-syntax (dispatcher-tree stx) ; the arguments, which are names of dispatcher variables @@ -26,11 +53,19 @@ (datum->syntax stx `(make-dispatcher-tree ,ds))) (define (make-dispatcher-tree ds) - (sequencer:make - (pathprocedure:make "/" (hash-ref ds 'page-home)) - (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) - (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) - (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) - (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) - (hash-ref ds 'static-dispatcher) - (lift:make (hash-ref ds 'page-not-found)))) + (host:make + (λ (host-sym) + (if/out (config-true? 'canonical_origin) + (let* ([host-header (symbol->string host-sym)] + [splitter (string-append "." (url-host (string->url (config-get 'canonical_origin))))] + [s (string-split host-header splitter #:trim? #f)]) + (if/in (and (eq? 2 (length s)) (equal? "" (cadr s))) + ((hash-ref ds 'subdomain-dispatcher) (car s)))) + (sequencer:make + (pathprocedure:make "/" (hash-ref ds 'page-home)) + (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) + (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) + (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) + (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) + (hash-ref ds 'static-dispatcher) + (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-home.rkt b/src/page-home.rkt index 102ab687..760ac17a 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -38,7 +38,7 @@ (define body `(html (head - (meta (@ (name ")viewport") (content "width=device-width, initial-scale=1"))) + (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) (title "About | BreezeWiki") (link (@ (rel "stylesheet") (type "text/css") (href "/static/internal.css"))) (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css")))) diff --git a/src/page-subdomain.rkt b/src/page-subdomain.rkt new file mode 100644 index 00000000..4028a78f --- /dev/null +++ b/src/page-subdomain.rkt @@ -0,0 +1,36 @@ +#lang racket/base +(require racket/path + racket/string + net/url + web-server/http + web-server/servlet-dispatch + html-writing + (prefix-in lift: web-server/dispatchers/dispatch-lift) + "config.rkt" + "xexpr-utils.rkt") + +(provide + subdomain-dispatcher) + +(define (subdomain-dispatcher subdomain) + (lift:make + (λ (req) + (response-handler + (define uri (request-uri req)) + (define path (url-path uri)) + (define path-string (string-join (map (λ (p) (path/param-path p)) path) "/")) + (define dest (format "~a/~a/~a" (config-get 'canonical_origin) subdomain path-string)) + (define dest-bytes (string->bytes/utf-8 dest)) + (response/output + #:code 302 + #:headers (list (header #"Location" dest-bytes)) + (λ (out) + (write-html + `(html + (head + (title "Redirecting...")) + (body + "Redirecting to " + (a (@ (href ,dest)) ,dest) + "...")) + out))))))) From 68832059bd121ea55d4c6bf7bcf4f988a4693e77 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 01:45:42 +1200 Subject: [PATCH 005/166] Display canonical_origin on the home page --- src/page-home.rkt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/page-home.rkt b/src/page-home.rkt index 760ac17a..27cdd2e4 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -1,6 +1,7 @@ #lang racket/base -(require html-writing +(require net/url + html-writing web-server/http "xexpr-utils.rkt" "config.rkt") @@ -31,7 +32,10 @@ ,(apply format "~a: ~a" x)))) examples)) (h2 "How to use") - (p "While browsing any page on Fandom, you can replace \"fandom.com\" in the address bar with \"breezewiki.com\" to see the BreezeWiki version of that page.") + (p ,(format "While browsing any page on Fandom, you can replace \"fandom.com\" in the address bar with \"~a\" to see the BreezeWiki version of that page." + (if (config-true? 'canonical_origin) + (url-host (string->url (config-get 'canonical_origin))) + "breezewiki.com"))) (p "After that, you can click the links to navigate around the pages.") (p "To get back to Fandom, click the link that's at the bottom of the page."))) From eabe699587bc58f046c0c461081ff3489e69d9b7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 01:46:29 +1200 Subject: [PATCH 006/166] Properly detect whether the instance is official --- src/page-home.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-home.rkt b/src/page-home.rkt index 27cdd2e4..89edd436 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -60,7 +60,7 @@ (footer (@ (class "custom-footer")) (div (@ (class "internal-footer")) (img (@ (class "my-logo") (src "/static/breezewiki.svg"))) - ,(if (config-get 'instance_is_official) + ,(if (config-true? 'instance_is_official) `(div (p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) (a (@ (href "https://cadence.moe/contact")) From 6be10ed31940a217c556934237684b74d09dca01 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 01:58:45 +1200 Subject: [PATCH 007/166] Unify application footer --- src/application-globals.rkt | 59 ++++++++++++++++++++++--------------- src/page-home.rkt | 19 ++---------- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index f7d9d67d..2d715f43 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -5,6 +5,8 @@ (provide ; timeout durations for http-easy requests timeouts + ; generates a consistent footer + application-footer ; generates a consistent template for wiki page content to sit in generate-wiki-page) @@ -14,6 +16,38 @@ (define timeouts (make-timeout-config #:lease 5 #:connect 5)) +(define (application-footer source-url) + `(footer (@ (class "custom-footer")) + (div (@ (class ,(if source-url "custom-footer__cols" "internal-footer"))) + (div (p + (img (@ (class "my-logo") (src "/static/breezewiki.svg")))) + (p + (a (@ (href "https://gitdab.com/cadence/breezewiki")) + ,(format "~a source code" (config-get 'application_name)))) + (p + (a (@ (href "https://docs.breezewiki.com")) + "Documentation and more information")) + (p + (a (@ (href "https://lists.sr.ht/~cadence/breezewiki-discuss")) + "Discussions / Bug reports / Feature requests")) + ,(if (config-true? 'instance_is_official) + `(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) + (a (@ (href "https://cadence.moe/contact")) + "Cadence.")) + `(p + ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name))))) + ,(if source-url + `(div (p "This page displays proxied content from " + (a (@ (href ,source-url) (rel "noreferrer")) ,source-url) + ". Text content is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " + (a (@ (href "https://www.fandom.com/licensing")) "see license info.") + " Media files may have different copying restrictions.") + (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))) + `(div (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " + (a (@ (href "https://www.fandom.com/licensing")) "see license info.") + " Media files and official Fandom documents have different copying restrictions.") + (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))))))) + (define (generate-wiki-page source-url wikiname title content) (define (required-styles origin) (map (λ (dest-path) (format dest-path origin)) @@ -46,29 +80,6 @@ (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,content)) - (footer (@ (class "custom-footer")) - (div (@ (class "custom-footer__cols")) - (div - (p - (img (@ (class "my-logo") (src "/static/breezewiki.svg")))) - (p - (a (@ (href "https://gitdab.com/cadence/breezewiki")) - ,(format "~a source code" (config-get 'application_name)))) - (p - (a (@ (href "https://lists.sr.ht/~cadence/breezewiki-discuss")) - "Discussions / Bug reports / Feature requests")) - ,(if (config-get 'instance_is_official) - `(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) - (a (@ (href "https://cadence.moe/contact")) - "Cadence.")) - `(p - ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name))))) - (div - (p "This page displays proxied content from " - (a (@ (href ,source-url) (rel "noreferrer")) ,source-url) - ". Text content is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " - (a (@ (href "https://www.fandom.com/licensing")) "see license info.") - " Media files may have different copying restrictions.") - (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))))))))))) + ,(application-footer source-url))))))) (module+ test (check-not-false (xexp->html (generate-wiki-page "" "test" "test" '(template))))) diff --git a/src/page-home.rkt b/src/page-home.rkt index 89edd436..26eb1aab 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -3,6 +3,7 @@ (require net/url html-writing web-server/http + "application-globals.rkt" "xexpr-utils.rkt" "config.rkt") @@ -57,23 +58,7 @@ (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,@content)) - (footer (@ (class "custom-footer")) - (div (@ (class "internal-footer")) - (img (@ (class "my-logo") (src "/static/breezewiki.svg"))) - ,(if (config-true? 'instance_is_official) - `(div - (p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) - (a (@ (href "https://cadence.moe/contact")) - "Cadence.")) - (p "Hosting generously provided by " - (a (@ (href "http://alphamethyl.barr0w.net/")) - "alphamethyl."))) - `(p - ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name)))) - (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " - (a (@ (href "https://www.fandom.com/licensing")) "see license info.") - " Media files and official Fandom documents have different copying restrictions.") - (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name))))))))))) + ,(application-footer #f))))))) (module+ test (check-not-false (xexp->html body))) From 44906015a5f4b6a69624dce55305f02aa895d4d2 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 13:34:16 +1200 Subject: [PATCH 008/166] Fix (& x) forms in update-tree --- src/xexpr-utils.rkt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/xexpr-utils.rkt b/src/xexpr-utils.rkt index 87f52fe1..5b87d340 100644 --- a/src/xexpr-utils.rkt +++ b/src/xexpr-utils.rkt @@ -158,7 +158,8 @@ (define attributes (bits->attributes (cdr element))) (define contents (filter element-is-content? (cdr element))) ; provide elements and strings (if (or (equal? element-type '*DECL) - (equal? element-type '@)) + (equal? element-type '@) + (equal? element-type '&)) ; special element, do nothing element ; regular element, transform it @@ -169,6 +170,10 @@ (map (λ (content) (if (element-is-element? content) (loop content) content)) contents))])))) +(module+ test + ; check (& x) sequences are preserved + (check-equal? (update-tree (λ (e t a c) (list t a c)) '(body "Hey" (& nbsp) (a (@ (href "/"))))) + '(body "Hey" (& nbsp) (a (@ (href "/")))))) (define (has-class? name attributes) (and (member name (string-split (or (get-attribute 'class attributes) "") " ")) #t)) From 9b3cd6dbe7c54d2bd23266060b955a090248d5ad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 13:38:16 +1200 Subject: [PATCH 009/166] Debug xexpr in page-wiki --- src/page-wiki.rkt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index ff86b203..f09900f1 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -13,6 +13,7 @@ web-server/http web-server/dispatchers/dispatch ; my libs + "config.rkt" "pure-utils.rkt" "xexpr-utils.rkt" "url-utils.rkt" @@ -231,12 +232,17 @@ (define body (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname))) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define headers (if redirect-msg + (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) + (list (header #"Refresh" value))) + (list))) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) (response/output #:code 200 - #:headers (if redirect-msg - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] - [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value))) - (list)) + #:headers headers (λ (out) (write-html body out))))))])) From 1f302b3a9ecd98a455966637d684c2a24275aaf8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 13:38:30 +1200 Subject: [PATCH 010/166] Add a pokemon example to the front page --- src/page-home.rkt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/page-home.rkt b/src/page-home.rkt index 26eb1aab..042d4ad1 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -15,6 +15,7 @@ (define examples '(("crosscode" "CrossCode_Wiki") + ("pokemon" "Eevee") ("minecraft" "Bricks") ("undertale" "Hot_Dog...%3F") ("tardis" "Eleanor_Blake") From 3505925f7ea85708eebaaf246728a742b0d93a7d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 5 Sep 2022 23:50:52 +1200 Subject: [PATCH 011/166] Increase table font size --- static/main.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/main.css b/static/main.css index b06c86e3..b2a3f3df 100644 --- a/static/main.css +++ b/static/main.css @@ -22,7 +22,7 @@ sub { } /* general page appearance */ -body.skin-fandomdesktop, button, input, textarea, .wikitable { +body.skin-fandomdesktop, button, input, textarea, .wikitable, .va-table { font-family: initial; font-size: 18px; line-height: 1.5; From 78399a3474d23289d7e4872e4f5a6d1ebf7eb13e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 8 Sep 2022 00:16:24 +1200 Subject: [PATCH 012/166] Add strict_proxy option to proxy links to images Default true. --- src/config.rkt | 3 ++- src/page-wiki.rkt | 35 +++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index 38433f95..4dc133ee 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -20,7 +20,8 @@ (canonical_origin . "") (debug . "false") (instance_is_official . "false") ; please don't turn this on, or you will make me very upset - (port . "10416"))) + (port . "10416") + (strict_proxy . "true"))) (define config (make-hasheq diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index f09900f1..c770abcf 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -30,9 +30,9 @@ (aside (@ (role "region") (class "portable-infobox pi-theme-wikia pi-layout-default")) (h2 (@ (class "pi-item pi-title") (data-source "title")) "Infobox Title") - (figure (@ (class "pi-item pi-image") (data-sourec "image")) - (a (@ (href "https://static.wiki.nocookie.net/nice-image.png") (class "image image-thumbnail") (title "")) - (img (@ (src "https://static.wiki.nocookie.net/nice-image-thumbnail.png") (class "pi-image-thumbnail"))))) + (figure (@ (class "pi-item pi-image") (data-source "image")) + (a (@ (href "https://static.wikia.nocookie.net/nice-image.png") (class "image image-thumbnail") (title "")) + (img (@ (src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") (class "pi-image-thumbnail"))))) (div (@ (class "pi-item pi-data") (data-source "description")) (h3 (@ (class "pi-data-label")) "Description") @@ -64,7 +64,7 @@ (check-equal? (preprocess-html-wiki "

Caption text.

") "
Caption text.
")) -(define (update-tree-wiki tree wikiname) +(define (update-tree-wiki tree wikiname #:strict-proxy? strict-proxy?) (update-tree (λ (element element-type attributes children) ;; replace whole element? @@ -142,7 +142,7 @@ (curry u (λ (v) (and (eq? element-type 'a) (has-class? "image" v))) - (λ (v) (dict-update attributes 'rel (λ (s) + (λ (v) (dict-update v 'rel (λ (s) (list (string-append (car s) " noreferrer"))) '("")))) ; proxy images from inline styles @@ -154,6 +154,13 @@ "url(" (u-proxy-url url) ")"))))) + ; and also their links, if strict-proxy is set + (curry u + (λ (v) + (and strict-proxy? + (eq? element-type 'a) + (has-class? "image-thumbnail" v))) + (λ (v) (attribute-maybe-update 'href u-proxy-url v))) ; proxy images from src attributes (curry attribute-maybe-update 'src u-proxy-url) ; don't lazyload images @@ -176,7 +183,7 @@ children))])) tree)) (module+ test - (define transformed (update-tree-wiki wiki-document "test")) + (define transformed (update-tree-wiki wiki-document "test" #:strict-proxy? #t)) ; check that wikilinks are changed to be local (check-equal? (get-attribute 'href (bits->attributes ((query-selector @@ -201,7 +208,19 @@ (check-equal? (let* ([alternative ((query-selector (λ (t a c) (has-class? "iframe-alternative" a)) transformed))] [link ((query-selector (λ (t a c) (eq? t 'a)) alternative))]) (get-attribute 'href (bits->attributes link))) - "https://example.com/iframe-src")) + "https://example.com/iframe-src") + ; check that images are proxied + (check-equal? (get-attribute 'src (bits->attributes + ((query-selector + (λ (t a c) (eq? t 'img)) + transformed)))) + "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image-thumbnail.png") + ; check that links to images are proxied + (check-equal? (get-attribute 'href (bits->attributes + ((query-selector + (λ (t a c) (and (eq? t 'a) (has-class? "image-thumbnail" a))) + transformed)))) + "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image.png")) (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) @@ -230,7 +249,7 @@ (next-dispatcher) (response-handler (define body - (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname))) + (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname #:strict-proxy? (config-true? 'strict_proxy)))) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] From e6eabe9cf409e7e6be7cc5bb4e6f6c959cf49533 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 8 Sep 2022 14:05:47 +1200 Subject: [PATCH 013/166] Proxy stylesheets too under strict_proxy This also refactors the configuration system to use make-parameter, because dynamic binding allows for testing code more easily. --- src/application-globals.rkt | 27 ++++++++++++--- src/config.rkt | 66 ++++++++++++++++++++++--------------- src/page-wiki.rkt | 12 ++++--- 3 files changed, 69 insertions(+), 36 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 2d715f43..11d610a4 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -1,6 +1,9 @@ #lang racket/base -(require net/http-easy - "config.rkt") +(require racket/string + net/http-easy + "config.rkt" + "xexpr-utils.rkt" + "url-utils.rkt") (provide ; timeout durations for http-easy requests @@ -50,7 +53,11 @@ (define (generate-wiki-page source-url wikiname title content) (define (required-styles origin) - (map (λ (dest-path) (format dest-path origin)) + (map (λ (dest-path) + (define url (format dest-path origin)) + (if (config-true? 'strict_proxy) + (u-proxy-url url) + url)) '(#;"~a/load.php?lang=en&modules=skin.fandomdesktop.styles&only=styles&skin=fandomdesktop" #;"~a/load.php?lang=en&modules=ext.gadget.dungeonsWiki%2CearthWiki%2Csite-styles%2Csound-styles&only=styles&skin=fandomdesktop" #;"~a/load.php?lang=en&modules=site.styles&only=styles&skin=fandomdesktop" @@ -82,4 +89,16 @@ ,content)) ,(application-footer source-url))))))) (module+ test - (check-not-false (xexp->html (generate-wiki-page "" "test" "test" '(template))))) + (define page + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (generate-wiki-page "" "test" "test" '(template)))) + ; check the page is a valid xexp + (check-not-false (xexp->html page)) + ; check the stylesheet is proxied + (check-true (string-prefix? + (get-attribute 'href + (bits->attributes + ((query-selector + (λ (t a c) (eq? t 'link)) + page)))) + "/proxy?dest=https%3A%2F%2Ftest.fandom.com"))) diff --git a/src/config.rkt b/src/config.rkt index 4dc133ee..16d1daff 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -4,16 +4,20 @@ ini) (provide + config-parameter config-true? config-get) (define-runtime-path path-config "../config.ini") +(define (config-parameter key) + (hash-ref config key)) + (define (config-true? key) - (not (member (hash-ref config key) '("" "false")))) + (not (member ((config-parameter key)) '("" "false")))) (define (config-get key) - (hash-ref config key)) + ((config-parameter key))) (define default-config '((application_name . "BreezeWiki") @@ -23,35 +27,43 @@ (port . "10416") (strict_proxy . "true"))) +(define loaded-alist + (with-handlers + ([exn:fail:filesystem:errno? + (λ (exn) + (begin0 + '() + (displayln "note: config file not detected, using defaults")))] + [exn:fail:contract? + (λ (exn) + (begin0 + '() + (displayln "note: config file empty or missing [] section, using defaults")))]) + (define l + (hash->list + (hash-ref + (ini->hash + (call-with-input-file path-config + (λ (in) + (read-ini in)))) + '||))) + (begin0 + l + (printf "note: ~a items loaded from config file~n" (length l))))) + +(define combined-alist (append default-config loaded-alist)) + (define config (make-hasheq - (append - default-config - (with-handlers - ([exn:fail:filesystem:errno? - (λ (exn) - (begin0 - '() - (displayln "note: config file not detected, using defaults")))] - [exn:fail:contract? - (λ (exn) - (begin0 - '() - (displayln "note: config file empty or missing [] section, using defaults")))]) - (define l - (hash->list - (hash-ref - (ini->hash - (call-with-input-file path-config - (λ (in) - (read-ini in)))) - '||))) - (begin0 - l - (printf "note: ~a items loaded from config file~n" (length l))))))) + (map (λ (pair) + (cons (car pair) (make-parameter (cdr pair)))) + combined-alist))) (when (config-true? 'debug) ; all values here are optimised for maximum prettiness (parameterize ([pretty-print-columns 80]) (display "config: ") - (pretty-write (hash->list config)))) + (pretty-write (sort + (hash->list (make-hasheq combined-alist)) + symbol

Caption text.

") "
Caption text.
")) -(define (update-tree-wiki tree wikiname #:strict-proxy? strict-proxy?) +(define (update-tree-wiki tree wikiname) (update-tree (λ (element element-type attributes children) ;; replace whole element? @@ -154,10 +154,10 @@ "url(" (u-proxy-url url) ")"))))) - ; and also their links, if strict-proxy is set + ; and also their links, if strict_proxy is set (curry u (λ (v) - (and strict-proxy? + (and (config-true? 'strict_proxy) (eq? element-type 'a) (has-class? "image-thumbnail" v))) (λ (v) (attribute-maybe-update 'href u-proxy-url v))) @@ -183,7 +183,9 @@ children))])) tree)) (module+ test - (define transformed (update-tree-wiki wiki-document "test" #:strict-proxy? #t)) + (define transformed + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (update-tree-wiki wiki-document "test"))) ; check that wikilinks are changed to be local (check-equal? (get-attribute 'href (bits->attributes ((query-selector @@ -249,7 +251,7 @@ (next-dispatcher) (response-handler (define body - (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname #:strict-proxy? (config-true? 'strict_proxy)))) + (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname))) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] From 0a27f6d87f0873dced3652c704835e788e0c5cfb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 8 Sep 2022 14:50:29 +1200 Subject: [PATCH 014/166] Improve homepage, add testimonials --- src/page-home.rkt | 21 ++++++++++++--------- static/internal.css | 4 ++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/page-home.rkt b/src/page-home.rkt index 042d4ad1..5374e0a0 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -24,22 +24,25 @@ (define content `((h2 "BreezeWiki makes wiki pages on Fandom readable") - (p "It removes ads, videos, and suggested content, leaving you with a clean page that doesn't consume all your data.") - (p "If you're looking for an \"alternative\" to Fandom for writing pages, you should look elsewhere. BreezeWiki only lets you read existing pages.") + (p "It removes ads, videos, and suggested content, leaving you with a clean page that doesn't slow down your device or use up your data.") (p "BreezeWiki can also be called an \"alternative frontend for Fandom\".") + (p ,(format "To use BreezeWiki, just replace \"fandom.com\" with \"~a\", and you'll instantly be teleported to a better world." + (if (config-true? 'canonical_origin) + (url-host (string->url (config-get 'canonical_origin))) + "breezewiki.com"))) (h2 "Example pages") (ul ,@(map (λ (x) `(li (a (@ (href ,(apply format "/~a/wiki/~a" x))) ,(apply format "~a: ~a" x)))) examples)) - (h2 "How to use") - (p ,(format "While browsing any page on Fandom, you can replace \"fandom.com\" in the address bar with \"~a\" to see the BreezeWiki version of that page." - (if (config-true? 'canonical_origin) - (url-host (string->url (config-get 'canonical_origin))) - "breezewiki.com"))) - (p "After that, you can click the links to navigate around the pages.") - (p "To get back to Fandom, click the link that's at the bottom of the page."))) + (h2 "Testimonials") + (p (@ (class "testimonial")) ">you are so right that fandom still sucks even with adblock somehow. even zapping all the stupid padding it still sucks —Minimus") + (p (@ (class "testimonial")) ">attempting to go to a wiki's forum page with breezewiki doesn't work, which is based honestly —Tom Skeleton") + (p (@ (class "testimonial")) ">Fandom pages crashing and closing, taking forever to load and locking up as they load the ads on the site... they are causing the site to crash because they are trying to load video ads both at the top and bottom of the site as well as two or three banner ads, then a massive top of site ad and eventually my anti-virus shuts the whole site down because it's literally pulling more resources than WoW in ultra settings... —Anonymous") + (h2 "What BreezeWiki isn't") + (p "BreezeWiki isn't an \"alternative\" to Fandom, and it doesn't let you edit or write new pages.") + (p "If you want to create your own wiki, try Miraheze!"))) (define body `(html diff --git a/static/internal.css b/static/internal.css index 6cc9294b..3b9a0376 100644 --- a/static/internal.css +++ b/static/internal.css @@ -171,3 +171,7 @@ body { .internal-footer { max-width: 700px; } + +.testimonial { + color: #157937; +} From 7a4bfe4180155e1388d9805e3e1e3b50fec284a1 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 15:42:20 +1200 Subject: [PATCH 015/166] Redirect /wikiname to its homepage --- breezewiki.rkt | 2 ++ src/application-globals.rkt | 26 +++++++++++++++++++++++--- src/dispatcher-tree.rkt | 1 + src/page-home.rkt | 1 + src/page-redirect-wiki-home.rkt | 15 +++++++++++++++ src/page-subdomain.rkt | 18 ++---------------- src/xexpr-utils.rkt | 5 +++-- 7 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 src/page-redirect-wiki-home.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index dbca4285..aeaaa890 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -13,6 +13,7 @@ (require-reloadable "src/page-home.rkt" page-home) (require-reloadable "src/page-not-found.rkt" page-not-found) (require-reloadable "src/page-proxy.rkt" page-proxy) +(require-reloadable "src/page-redirect-wiki-home.rkt" redirect-wiki-home) (require-reloadable "src/page-search.rkt" page-search) (require-reloadable "src/page-static.rkt" static-dispatcher) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) @@ -35,6 +36,7 @@ page-proxy page-search page-wiki + redirect-wiki-home static-dispatcher subdomain-dispatcher)))) (define server-t (thread start)) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 11d610a4..b47ed309 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -1,6 +1,8 @@ #lang racket/base (require racket/string - net/http-easy + (prefix-in easy: net/http-easy) + html-writing + web-server/http "config.rkt" "xexpr-utils.rkt" "url-utils.rkt") @@ -11,13 +13,15 @@ ; generates a consistent footer application-footer ; generates a consistent template for wiki page content to sit in - generate-wiki-page) + generate-wiki-page + ; generates a minimal but complete redirect to another page + generate-redirect) (module+ test (require rackunit html-writing)) -(define timeouts (make-timeout-config #:lease 5 #:connect 5)) +(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define (application-footer source-url) `(footer (@ (class "custom-footer")) @@ -102,3 +106,19 @@ (λ (t a c) (eq? t 'link)) page)))) "/proxy?dest=https%3A%2F%2Ftest.fandom.com"))) + +(define (generate-redirect dest) + (define dest-bytes (string->bytes/utf-8 dest)) + (response/output + #:code 302 + #:headers (list (header #"Location" dest-bytes)) + (λ (out) + (write-html + `(html + (head + (title "Redirecting...")) + (body + "Redirecting to " + (a (@ (href ,dest)) ,dest) + "...")) + out)))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 49306d8a..58e8eae0 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -67,5 +67,6 @@ (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) + (filter:make #rx"^/[a-z-]+(/(wiki(/)?)?)?$" (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-home.rkt b/src/page-home.rkt index 5374e0a0..7e4184c8 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -37,6 +37,7 @@ ,(apply format "~a: ~a" x)))) examples)) (h2 "Testimonials") + (p (@ (class "testimonial")) ">So glad to never have to touch fandom's garbage platform directly ever again —RNL") (p (@ (class "testimonial")) ">you are so right that fandom still sucks even with adblock somehow. even zapping all the stupid padding it still sucks —Minimus") (p (@ (class "testimonial")) ">attempting to go to a wiki's forum page with breezewiki doesn't work, which is based honestly —Tom Skeleton") (p (@ (class "testimonial")) ">Fandom pages crashing and closing, taking forever to load and locking up as they load the ads on the site... they are causing the site to crash because they are trying to load video ads both at the top and bottom of the site as well as two or three banner ads, then a massive top of site ad and eventually my anti-virus shuts the whole site down because it's literally pulling more resources than WoW in ultra settings... —Anonymous") diff --git a/src/page-redirect-wiki-home.rkt b/src/page-redirect-wiki-home.rkt new file mode 100644 index 00000000..a3740b52 --- /dev/null +++ b/src/page-redirect-wiki-home.rkt @@ -0,0 +1,15 @@ +#lang racket/base +(require net/url + web-server/http + "application-globals.rkt" + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide + redirect-wiki-home) + +(define (redirect-wiki-home req) + (response-handler + (define wikiname (path/param-path (car (url-path (request-uri req))))) + (define dest (format "~a/wiki/Main_Page" wikiname)) + (generate-redirect dest))) diff --git a/src/page-subdomain.rkt b/src/page-subdomain.rkt index 4028a78f..dededfb1 100644 --- a/src/page-subdomain.rkt +++ b/src/page-subdomain.rkt @@ -3,9 +3,8 @@ racket/string net/url web-server/http - web-server/servlet-dispatch - html-writing (prefix-in lift: web-server/dispatchers/dispatch-lift) + "application-globals.rkt" "config.rkt" "xexpr-utils.rkt") @@ -20,17 +19,4 @@ (define path (url-path uri)) (define path-string (string-join (map (λ (p) (path/param-path p)) path) "/")) (define dest (format "~a/~a/~a" (config-get 'canonical_origin) subdomain path-string)) - (define dest-bytes (string->bytes/utf-8 dest)) - (response/output - #:code 302 - #:headers (list (header #"Location" dest-bytes)) - (λ (out) - (write-html - `(html - (head - (title "Redirecting...")) - (body - "Redirecting to " - (a (@ (href ,dest)) ,dest) - "...")) - out))))))) + (generate-redirect dest))))) diff --git a/src/xexpr-utils.rkt b/src/xexpr-utils.rkt index 5b87d340..cd7f6517 100644 --- a/src/xexpr-utils.rkt +++ b/src/xexpr-utils.rkt @@ -197,6 +197,7 @@ (λ (out) (for ([port (list (current-error-port) out)]) (parameterize ([current-error-port port]) - (displayln "Exception raised in Racket code at response generation time:" (current-error-port)) - ((error-display-handler) (exn-message e) e))))))]) + (with-handlers ([exn:fail? (λ (e) (void))]) + (displayln "Exception raised in Racket code at response generation time:" (current-error-port)) + ((error-display-handler) (exn-message e) e)))))))]) body ...)) From 31037c98527b24387b525fee3572a0d4f0943be7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 16:24:26 +1200 Subject: [PATCH 016/166] fix previous commit --- dist.rkt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist.rkt b/dist.rkt index d0720be1..596a1b97 100644 --- a/dist.rkt +++ b/dist.rkt @@ -7,6 +7,7 @@ (require (only-in "src/page-home.rkt" page-home)) (require (only-in "src/page-not-found.rkt" page-not-found)) (require (only-in "src/page-proxy.rkt" page-proxy)) +(require (only-in "src/page-redirect-wiki-home.rkt" redirect-wiki-home)) (require (only-in "src/page-search.rkt" page-search)) (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) @@ -24,5 +25,6 @@ page-proxy page-search page-wiki + redirect-wiki-home static-dispatcher subdomain-dispatcher))) From 62e1cb33c865bf757cf503b3597a042fbb1628fa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 21:59:19 +1200 Subject: [PATCH 017/166] Prevent footer overlapping on mobile --- static/main.css | 1 + 1 file changed, 1 insertion(+) diff --git a/static/main.css b/static/main.css index b2a3f3df..63dc408b 100644 --- a/static/main.css +++ b/static/main.css @@ -55,6 +55,7 @@ p { /* custom footer with source and license info */ .custom-footer { + clear: both; font-size: 16px; margin-top: 30px; padding-top: 20px; From 660ec56e7337b20aedde2e15c10145168b5eb69f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 23:17:21 +1200 Subject: [PATCH 018/166] Mobile layout improvements - Images must fit within the screen - Attempt to make horizontal page footers vertical on mobile --- static/main.css | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/static/main.css b/static/main.css index 63dc408b..fe8500ac 100644 --- a/static/main.css +++ b/static/main.css @@ -64,11 +64,6 @@ p { .custom-footer__cols { display: grid; } -@media (min-width: 560px) { - .custom-footer__cols { - grid-template-columns: 1fr 1fr; - } -} .my-logo { margin-bottom: 8px; } @@ -115,6 +110,13 @@ p { display: none; } +/* don't allow images to extend past the edge of the page */ +img { + max-width: 100%; + width: auto; + height: auto; +} + /* allow tables to scroll horizontally if needed */ .table-scroller { overflow: auto; @@ -178,13 +180,6 @@ figcaption, .lightbox-caption, .thumbcaption { display: block; } -/* category list columns */ -@media (min-width: 700px) { - .my-category-list { - columns: 3; - } -} - /* animated slots */ #mw-content-text .animated > :not(.animated-active), #mw-content-text .animated > .animated-subframe > :not(.animated-active) { display: inline-block; @@ -221,3 +216,29 @@ figcaption, .lightbox-caption, .thumbcaption { color: var(--theme-page-text-color--hover); margin-left: 1.2em; } + +/* media queries */ + +/* for reference, cell phone screens are generally 400 px wide, definitely less than 500 px */ + +@media (min-width: 700px) { /* wider than 700 px */ + /* category list columns */ + .my-category-list { + columns: 3; + } +} + +@media (min-width: 560px) { /* wider than 560 px */ + /* footer columns */ + .custom-footer__cols { + grid-template-columns: 1fr 1fr; + } +} + +@media (max-width: 559px) { /* narrower than 560 px */ + /* attempt to make horizontal wiki page footers vertical instead - see /nfs/wiki/Need_for_Speed_Wiki */ + [style*="width:calc(100%"] { + width: auto !important; + text-align: center !important; + } +} From 229ec1ee0c6622594caa123c42718a7dd4d5c959 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 23:33:01 +1200 Subject: [PATCH 019/166] Don't display new editor encouragement --- static/main.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/static/main.css b/static/main.css index fe8500ac..2aa449eb 100644 --- a/static/main.css +++ b/static/main.css @@ -110,6 +110,11 @@ p { display: none; } +/* don't display sections on home pages encouraging new editors */ +#fpcommunity { + display: none; +} + /* don't allow images to extend past the edge of the page */ img { max-width: 100%; From 951a5fe651e4005172e3720d48420fb17711f093 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Sep 2022 23:41:33 +1200 Subject: [PATCH 020/166] Fix wiki home redirection --- src/page-redirect-wiki-home.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-redirect-wiki-home.rkt b/src/page-redirect-wiki-home.rkt index a3740b52..b0c3df3c 100644 --- a/src/page-redirect-wiki-home.rkt +++ b/src/page-redirect-wiki-home.rkt @@ -11,5 +11,5 @@ (define (redirect-wiki-home req) (response-handler (define wikiname (path/param-path (car (url-path (request-uri req))))) - (define dest (format "~a/wiki/Main_Page" wikiname)) + (define dest (format "/~a/wiki/Main_Page" wikiname)) (generate-redirect dest))) From e4bc962b05329009dd424f16399c5fea794f503f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 11 Sep 2022 19:05:34 +1200 Subject: [PATCH 021/166] Link to automatic redirection tutorial on homepage --- src/page-home.rkt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/page-home.rkt b/src/page-home.rkt index 7e4184c8..3c84511f 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -30,6 +30,8 @@ (if (config-true? 'canonical_origin) (url-host (string->url (config-get 'canonical_origin))) "breezewiki.com"))) + (p "If you'd like to be automatically sent to BreezeWiki every time in the future, " + (a (@ (href "https://docs.breezewiki.com/Automatic_Redirection.html")) "check out the tutorial in the manual.")) (h2 "Example pages") (ul ,@(map (λ (x) From 4ad22ca9c1a7b65369236350c76c4f7bd2101498 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 11 Sep 2022 19:38:20 +1200 Subject: [PATCH 022/166] Move syntax definitions to a new file --- src/dispatcher-tree.rkt | 26 ++------------------ src/syntax.rkt | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 src/syntax.rkt diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 58e8eae0..5800876d 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -1,5 +1,6 @@ #lang racket/base -(require (for-syntax racket/base) +(require "syntax.rkt" + (for-syntax racket/base) racket/string net/url (prefix-in host: web-server/dispatchers/dispatch-host) @@ -15,29 +16,6 @@ ; procedure to make the tree from the hashmap make-dispatcher-tree) -(define-syntax (if/out stx) - (define tree (cdr (syntax->datum stx))) ; condition true false - (define else (cddr tree)) ; the else branch cons cell - (define result - (let walk ([node tree]) - (cond - ; normally, node should be a full cons cell (a pair) but it might be something else. - ; situation: reached the end of a list, empty cons cell - [(null? node) node] - ; situation: reached the end of a list, cons cdr was non-list - [(symbol? node) node] - ; normal situation, full cons cell - ; -- don't go replacing through nested if/out - [(and (pair? node) (eq? 'if/out (car node))) node] - ; -- replace if/in - [(and (pair? node) (eq? 'if/in (car node))) - (append '(if) (cdr node) else)] - ; recurse down pair head and tail - [(pair? node) (cons (walk (car node)) (walk (cdr node)))] - ; something else that can't be recursed into, so pass it through - [#t node]))) - (datum->syntax stx (cons 'if result))) - ; make a hashmap out of the provided names and call make-dispatcher-tree with it (define-syntax (dispatcher-tree stx) ; the arguments, which are names of dispatcher variables diff --git a/src/syntax.rkt b/src/syntax.rkt new file mode 100644 index 00000000..c331493a --- /dev/null +++ b/src/syntax.rkt @@ -0,0 +1,54 @@ +#lang racket/base +(require (for-syntax racket/base)) + +(provide + ; help make a nested if where the false results are the same + if/out) + +(module+ test + (require rackunit) + (define (check-syntax-equal? s1 s2) + (check-equal? (syntax->datum s1) + (syntax->datum s2)))) + +;; actual transforming goes on in here. +;; it's in a submodule so that it can be required in both levels, for testing + +(module transform racket/base + (provide transform-if/out) + (define (transform-if/out stx) + (define tree (cdr (syntax->datum stx))) ; condition true false + (define else (cddr tree)) ; the else branch cons cell + (define result + (let walk ([node tree]) + (cond + ; normally, node should be a full cons cell (a pair) but it might be something else. + ; situation: reached the end of a list, empty cons cell + [(null? node) node] + ; situation: reached the end of a list, cons cdr was non-list + [(symbol? node) node] + ; normal situation, full cons cell + ; -- don't go replacing through nested if/out + [(and (pair? node) (eq? 'if/out (car node))) node] + ; -- replace if/in + [(and (pair? node) (eq? 'if/in (car node))) + (append '(if) (cdr node) else)] + ; recurse down pair head and tail + [(pair? node) (cons (walk (car node)) (walk (cdr node)))] + ; something else that can't be recursed into, so pass it through + [#t node]))) + (datum->syntax stx (cons 'if result)))) + +;; the syntax definitions and their tests go below here + +(require 'transform (for-syntax 'transform)) + +(define-syntax (if/out stx) + (transform-if/out stx)) +(module+ test + (check-syntax-equal? (transform-if/out #'(if/out (condition 1) (if/in (condition 2) (do-yes)) (do-no))) + #'(if (condition 1) (if (condition 2) (do-yes) (do-no)) (do-no))) + (check-equal? (if/out #t (if/in #t 'yes) 'no) 'yes) + (check-equal? (if/out #f (if/in #t 'yes) 'no) 'no) + (check-equal? (if/out #t (if/in #f 'yes) 'no) 'no) + (check-equal? (if/out #f (if/in #f 'yes) 'no) 'no)) From 33ee6a0624f86053f670decf2b01fbe762e556e8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 11 Sep 2022 23:21:37 +1200 Subject: [PATCH 023/166] Also display the content on category pages --- src/application-globals.rkt | 5 +- src/page-category.rkt | 93 +++++++++++++++++++++++-------------- src/page-wiki.rkt | 10 ++-- src/syntax.rkt | 62 +++++++++++++++++++++++-- 4 files changed, 125 insertions(+), 45 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index b47ed309..d5d592d3 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -66,13 +66,12 @@ #;"~a/load.php?lang=en&modules=ext.gadget.dungeonsWiki%2CearthWiki%2Csite-styles%2Csound-styles&only=styles&skin=fandomdesktop" #;"~a/load.php?lang=en&modules=site.styles&only=styles&skin=fandomdesktop" ; combine the above entries into a single request for potentially extra speed - fandom.com doesn't even do this! - "~a/load.php?lang=en&modules=skin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles%7Csite.styles&only=styles&skin=fandomdesktop" - "~a/wikia.php?controller=ThemeApi&method=themeVariables"))) + "~a/wikia.php?controller=ThemeApi&method=themeVariables" + "~a/load.php?lang=en&modules=skin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles%7Csite.styles&only=styles&skin=fandomdesktop"))) `(html (head (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) (title ,(format "~a | ~a" title (config-get 'application_name))) - (style ":root { --theme-page-background-color: #dfdfe0 }") ; fallback in case styles don't load fast enough ,@(map (λ (url) `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) (required-styles (format "https://~a.fandom.com" wikiname))) diff --git a/src/page-category.rkt b/src/page-category.rkt index bfc689fa..bb388dab 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -4,14 +4,17 @@ racket/string (prefix-in easy: net/http-easy) ; html libs + html-parsing html-writing ; web server libs net/url web-server/http (only-in web-server/dispatchers/dispatch next-dispatcher) #;(only-in web-server/http/redirect redirect-to) - "config.rkt" "application-globals.rkt" + "config.rkt" + "page-wiki.rkt" + "syntax.rkt" "url-utils.rkt" "xexpr-utils.rkt") @@ -23,50 +26,70 @@ (define category-json-data '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) -(define (generate-results-page dest-url wikiname prefixed-category data) - (define members (jp "/query/categorymembers" data)) +(define (generate-results-page dest-url wikiname prefixed-category members-data page) + (define members (jp "/query/categorymembers" members-data)) (generate-wiki-page dest-url wikiname prefixed-category - `(div (@ (class "mw-parser-output")) - (ul (@ (class "my-category-list")) - ,@(map - (λ (result) - (define title (jp "/title" result)) - (define page-path (regexp-replace* #rx" " title "_")) - `(li - (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title))) - members))))) + `(div + ,(update-tree-wiki page wikiname) + (hr) + (h2 ,(format "All Pages in ~a" prefixed-category)) + (div (@ (class "mw-parser-output")) + (ul (@ (class "my-category-list")) + ,@(map + (λ (result) + (define title (jp "/title" result)) + (define page-path (regexp-replace* #rx" " title "_")) + `(li + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title))) + members)))))) (define (page-category req) (response-handler (define wikiname (path/param-path (first (url-path (request-uri req))))) (define prefixed-category (path/param-path (caddr (url-path (request-uri req))))) - (define origin (format "https://~a.fandom.com" wikiname)) - (define dest-url (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (printf "out: ~a~n" dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) + (define source-url (format "~a/wiki/~a" origin prefixed-category)) - (define data (easy:response-json dest-res)) - (define body (generate-results-page dest-url wikiname prefixed-category data)) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - (λ (out) - (write-html body out))))) + (thread-let + ([members-data (define dest-url (format "~a/api.php?~a" + origin + (params->query `(("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))))) + (printf "out: ~a~n" dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [page-data (define dest-url (format "~a/api.php?~a" + origin + (params->query `(("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text") + ("formatversion" . "2") + ("format" . "json"))))) + (printf "out: ~a~n" dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)]) + + (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) + (define page (html->xexp page-html)) + (define body (generate-results-page source-url wikiname prefixed-category members-data page)) + + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + (λ (out) + (write-html body out)))))) (module+ test (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") - (generate-results-page "" "test" "Category:Items" category-json-data))))) + (generate-results-page "" "test" "Category:Items" category-json-data + '(div "page text")))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 65771a8a..bcf13aaa 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -13,14 +13,18 @@ web-server/http web-server/dispatchers/dispatch ; my libs + "application-globals.rkt" "config.rkt" "pure-utils.rkt" "xexpr-utils.rkt" - "url-utils.rkt" - "application-globals.rkt") + "url-utils.rkt") (provide - page-wiki) + ; used by the web server + page-wiki + ; used by page-category, and similar pages that are partially wiki pages + update-tree-wiki + preprocess-html-wiki) (module+ test (require rackunit) diff --git a/src/syntax.rkt b/src/syntax.rkt index c331493a..8205326b 100644 --- a/src/syntax.rkt +++ b/src/syntax.rkt @@ -2,8 +2,10 @@ (require (for-syntax racket/base)) (provide - ; help make a nested if where the false results are the same - if/out) + ; help make a nested if. if/in will gain the same false form of its containing if/out. + if/out + ; let, but the value for each variable is evaluated within a thread + thread-let) (module+ test (require rackunit) @@ -15,7 +17,10 @@ ;; it's in a submodule so that it can be required in both levels, for testing (module transform racket/base - (provide transform-if/out) + (provide + transform-if/out + transform-thread-let) + (define (transform-if/out stx) (define tree (cdr (syntax->datum stx))) ; condition true false (define else (cddr tree)) ; the else branch cons cell @@ -37,7 +42,27 @@ [(pair? node) (cons (walk (car node)) (walk (cdr node)))] ; something else that can't be recursed into, so pass it through [#t node]))) - (datum->syntax stx (cons 'if result)))) + (datum->syntax stx (cons 'if result))) + + (define (transform-thread-let stx) + (define tree (cdr (syntax->datum stx))) + (define defs (car tree)) + (define forms (cdr tree)) + (when (eq? (length forms) 0) + (error (format "thread-let: bad syntax (need some forms to execute after the threads)~n forms: ~a" forms))) + (define counter (build-list (length defs) values)) + (datum->syntax + stx + `(let ([chv (build-vector ,(length defs) (λ (_) (make-channel)))]) + ,@(map (λ (n) + (define def (list-ref defs n)) + `(thread (λ () (channel-put (vector-ref chv ,n) (let _ () ,@(cdr def)))))) + counter) + (let ,(map (λ (n) + (define def (list-ref defs n)) + `(,(car def) (channel-get (vector-ref chv ,n)))) + counter) + ,@forms))))) ;; the syntax definitions and their tests go below here @@ -52,3 +77,32 @@ (check-equal? (if/out #f (if/in #t 'yes) 'no) 'no) (check-equal? (if/out #t (if/in #f 'yes) 'no) 'no) (check-equal? (if/out #f (if/in #f 'yes) 'no) 'no)) + +(define-syntax (thread-let stx) + (transform-thread-let stx)) +(module+ test + ; check that it is transformed as expected + (check-syntax-equal? + (transform-thread-let + #'(thread-let ([a (hey "this is a")] + [b (hey "this is b")]) + (list a b))) + #'(let ([chv (build-vector 2 (λ (_) (make-channel)))]) + (thread (λ () (channel-put (vector-ref chv 0) (let _ () (hey "this is a"))))) + (thread (λ () (channel-put (vector-ref chv 1) (let _ () (hey "this is b"))))) + (let ([a (channel-get (vector-ref chv 0))] + [b (channel-get (vector-ref chv 1))]) + (list a b)))) + ; check that they actually execute concurrently + (define ch (make-channel)) + (check-equal? (thread-let ([a (begin + (channel-put ch 'a) + (channel-get ch))] + [b (begin0 + (channel-get ch) + (channel-put ch 'b))]) + (list a b)) + '(b a)) + ; check that it assigns the correct value to the correct variable + (check-equal? (thread-let ([a (sleep 0) 'a] [b 'b]) (list a b)) + '(a b))) From 0172034319b6b1a51ae2f610f7a16e51b78f5d62 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 00:56:05 +1200 Subject: [PATCH 024/166] Match original light/dark page theme reliably - Use keyword parameters for generate-wiki-page - Apply body classes (including page theme) from original source --- src/application-globals.rkt | 18 ++++++-- src/dispatcher-tree.rkt | 4 +- src/page-category.rkt | 82 ++++++++++++++++++++++++------------- src/page-search.rkt | 6 +-- src/page-wiki.rkt | 16 ++++++-- 5 files changed, 86 insertions(+), 40 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index d5d592d3..d21858b7 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -55,7 +55,15 @@ " Media files and official Fandom documents have different copying restrictions.") (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))))))) -(define (generate-wiki-page source-url wikiname title content) +(define (generate-wiki-page + content + #:source-url source-url + #:wikiname wikiname + #:title title + #:body-class [body-class-in ""]) + (define body-class (if (equal? "" body-class-in) + "skin-fandomdesktop" + body-class-in)) (define (required-styles origin) (map (λ (dest-path) (define url (format dest-path origin)) @@ -76,7 +84,7 @@ `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) (required-styles (format "https://~a.fandom.com" wikiname))) (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css")))) - (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light")) + (body (@ (class ,body-class)) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileHorizontally header"))) (div (@ (class "page")) @@ -94,7 +102,11 @@ (module+ test (define page (parameterize ([(config-parameter 'strict_proxy) "true"]) - (generate-wiki-page "" "test" "test" '(template)))) + (generate-wiki-page + '(template) + #:source-url "" + #:title "test" + #:wikiname "test"))) ; check the page is a valid xexp (check-not-false (xexp->html page)) ; check the stylesheet is proxied diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 5800876d..13321bdd 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -12,9 +12,7 @@ (provide ; syntax to make the hashmap from names - dispatcher-tree - ; procedure to make the tree from the hashmap - make-dispatcher-tree) + dispatcher-tree) ; make a hashmap out of the provided names and call make-dispatcher-tree with it (define-syntax (dispatcher-tree stx) diff --git a/src/page-category.rkt b/src/page-category.rkt index bb388dab..0bb9080e 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -1,6 +1,7 @@ #lang racket/base (require racket/dict racket/list + racket/match racket/string (prefix-in easy: net/http-easy) ; html libs @@ -26,12 +27,19 @@ (define category-json-data '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) -(define (generate-results-page dest-url wikiname prefixed-category members-data page) +(define (generate-results-page + #:source-url source-url + #:wikiname wikiname + #:prefixed-category prefixed-category + #:members-data members-data + #:page page + #:body-class body-class) (define members (jp "/query/categorymembers" members-data)) (generate-wiki-page - dest-url - wikiname - prefixed-category + #:source-url source-url + #:wikiname wikiname + #:title prefixed-category + #:body-class body-class `(div ,(update-tree-wiki page wikiname) (hr) @@ -55,31 +63,45 @@ (define source-url (format "~a/wiki/~a" origin prefixed-category)) (thread-let - ([members-data (define dest-url (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (printf "out: ~a~n" dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)] - [page-data (define dest-url (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,prefixed-category) - ("prop" . "text") - ("formatversion" . "2") - ("format" . "json"))))) - (printf "out: ~a~n" dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)]) + ([members-data (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))))) + (printf "out: ~a~n" dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [page-data (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))))) + (printf "out: ~a~n" dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)]) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) (define page (html->xexp page-html)) - (define body (generate-results-page source-url wikiname prefixed-category members-data page)) + (define head-html (jp "/parse/headhtml" page-data "")) + (define body-class (match (regexp-match #rx"]*class=\"([^\"]*)" head-html) + [(list _ classes) classes] + [_ ""])) + (println head-html) + (println body-class) + (define body (generate-results-page + #:source-url source-url + #:wikiname wikiname + #:prefixed-category prefixed-category + #:members-data members-data + #:page page + #:body-class body-class)) (when (config-true? 'debug) ; used for its side effects @@ -91,5 +113,9 @@ (write-html body out)))))) (module+ test (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") - (generate-results-page "" "test" "Category:Items" category-json-data - '(div "page text")))))) + (generate-results-page + #:source-url "" + #:wikiname "test" + #:prefixed-category "Category:Items" + #:category-data category-json-data + #:page '(div "page text")))))) diff --git a/src/page-search.rkt b/src/page-search.rkt index 7b0abe39..5d1bd7df 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -26,9 +26,9 @@ (define (generate-results-page dest-url wikiname query data) (define search-results (jp "/query/search" data)) (generate-wiki-page - dest-url - wikiname - "Search Results" + #:source-url dest-url + #:wikiname wikiname + #:title "Search Results" `(div (@ (class "mw-parser-output")) (p ,(format "~a results found for " (length search-results)) (strong ,query)) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index bcf13aaa..ecfdbb35 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -2,6 +2,7 @@ (require racket/dict racket/function racket/list + racket/match racket/string ; libs (prefix-in easy: net/http-easy) @@ -238,7 +239,7 @@ origin (params->query `(("action" . "parse") ("page" . ,path) - ("prop" . "text") + ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json"))))) (printf "out: ~a~n" dest-url) @@ -250,12 +251,21 @@ [title (jp "/parse/title" data "")] [page-html (jp "/parse/text" data "")] [page-html (preprocess-html-wiki page-html)] - [page (html->xexp page-html)]) + [page (html->xexp page-html)] + [head-html (jp "/parse/headhtml" data "")] + [body-class (match (regexp-match #rx"]*class=\"([^\"]*)" head-html) + [(list _ classes) classes] + [_ ""])]) (if (equal? "missingtitle" (jp "/error/code" data #f)) (next-dispatcher) (response-handler (define body - (generate-wiki-page source-url wikiname title (update-tree-wiki page wikiname))) + (generate-wiki-page + (update-tree-wiki page wikiname) + #:source-url source-url + #:wikiname wikiname + #:title title + #:body-class body-class)) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] From 37318b8c50b640ce061163f6130a6b86f3e4968e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 00:58:06 +1200 Subject: [PATCH 025/166] Style updates - Fix code block font - Use sans-serif font (don't know how I missed this) - Smaller font size on cell phones --- static/main.css | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/static/main.css b/static/main.css index 2aa449eb..7ef51a13 100644 --- a/static/main.css +++ b/static/main.css @@ -1,11 +1,17 @@ /* reset the reset */ -blockquote, code, del, details, div, dl, dt, em, fieldset, figcaption, figure, h1, h2, h3, h4, h5, h6, li -, ol, p, pre, q, span, strong, ul { - font-family: initial; +blockquote, code, del, details, div, dl, dt, em, fieldset, figcaption, figure, h1, h2, h3, h4, h5, h6, li, ol, p, pre, q, span, strong, ul { + font-family: sans-serif; margin: initial; padding: initial; border: initial; } +input, textarea { + font-family: sans-serif; +} +pre, code { + font-family: monospace; + font-size: 0.85em; +} ul, ol { list-style-type: initial; padding-left: 2em; @@ -23,7 +29,6 @@ sub { /* general page appearance */ body.skin-fandomdesktop, button, input, textarea, .wikitable, .va-table { - font-family: initial; font-size: 18px; line-height: 1.5; } @@ -233,6 +238,13 @@ figcaption, .lightbox-caption, .thumbcaption { } } +@media (max-width: 699px) { /* narrower than 700 px */ + /* fit more text on screen */ + body.skin-fandomdesktop, button, input, textarea, .wikitable, .va-table { + font-size: 16px; + } +} + @media (min-width: 560px) { /* wider than 560 px */ /* footer columns */ .custom-footer__cols { From a9acfc34a255226c745d106cf028ac651ad6f1f6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 01:56:03 +1200 Subject: [PATCH 026/166] Display correct license information --- src/application-globals.rkt | 15 +++--- src/data.rkt | 33 +++++++++++++ src/dispatcher-tree.rkt | 4 +- src/page-category.rkt | 15 +++--- src/page-search.rkt | 50 ++++++++++--------- src/page-wiki.rkt | 98 ++++++++++++++++++++----------------- 6 files changed, 134 insertions(+), 81 deletions(-) create mode 100644 src/data.rkt diff --git a/src/application-globals.rkt b/src/application-globals.rkt index d21858b7..c2149246 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -4,6 +4,7 @@ html-writing web-server/http "config.rkt" + "data.rkt" "xexpr-utils.rkt" "url-utils.rkt") @@ -23,7 +24,8 @@ (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) -(define (application-footer source-url) +(define (application-footer source-url #:license [license-in #f]) + (define license (or license-in license-default)) `(footer (@ (class "custom-footer")) (div (@ (class ,(if source-url "custom-footer__cols" "internal-footer"))) (div (p @@ -46,8 +48,8 @@ ,(if source-url `(div (p "This page displays proxied content from " (a (@ (href ,source-url) (rel "noreferrer")) ,source-url) - ". Text content is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " - (a (@ (href "https://www.fandom.com/licensing")) "see license info.") + ,(format ". Text content is available under the ~a license, " (license-text license)) + (a (@ (href ,(license-url license))) "see license info.") " Media files may have different copying restrictions.") (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))) `(div (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " @@ -60,8 +62,9 @@ #:source-url source-url #:wikiname wikiname #:title title - #:body-class [body-class-in ""]) - (define body-class (if (equal? "" body-class-in) + #:body-class [body-class-in #f] + #:license [license #f]) + (define body-class (if (not body-class-in) "skin-fandomdesktop" body-class-in)) (define (required-styles origin) @@ -98,7 +101,7 @@ (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,content)) - ,(application-footer source-url))))))) + ,(application-footer source-url #:license license))))))) (module+ test (define page (parameterize ([(config-parameter 'strict_proxy) "true"]) diff --git a/src/data.rkt b/src/data.rkt new file mode 100644 index 00000000..f7a0d1f4 --- /dev/null +++ b/src/data.rkt @@ -0,0 +1,33 @@ +#lang racket/base +(require (prefix-in easy: net/http-easy) + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide + (struct-out license) + license-default + license-auto) + +(struct license (text url) #:transparent) +(define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing")) +(define license-hash (make-hash)) +(define (license-fetch wikiname) + (define dest-url + (format "https://~a.fandom.com/api.php?~a" + wikiname + (params->query '(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "rightsinfo") + ("format" . "json") + ("formatversion" . "2"))))) + (printf "out: ~a~n" dest-url) + (define res (easy:get dest-url)) + (define data (easy:response-json res)) + (license (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data))) +(define (license-auto wikiname) + (if (hash-has-key? license-hash wikiname) + (hash-ref license-hash wikiname) + (let ([result (license-fetch wikiname)]) + (hash-set! license-hash wikiname result) + result))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 13321bdd..5800876d 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -12,7 +12,9 @@ (provide ; syntax to make the hashmap from names - dispatcher-tree) + dispatcher-tree + ; procedure to make the tree from the hashmap + make-dispatcher-tree) ; make a hashmap out of the provided names and call make-dispatcher-tree with it (define-syntax (dispatcher-tree stx) diff --git a/src/page-category.rkt b/src/page-category.rkt index 0bb9080e..8ecd3689 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -14,6 +14,7 @@ #;(only-in web-server/http/redirect redirect-to) "application-globals.rkt" "config.rkt" + "data.rkt" "page-wiki.rkt" "syntax.rkt" "url-utils.rkt" @@ -33,13 +34,15 @@ #:prefixed-category prefixed-category #:members-data members-data #:page page - #:body-class body-class) + #:body-class [body-class #f] + #:license [license #f]) (define members (jp "/query/categorymembers" members-data)) (generate-wiki-page #:source-url source-url #:wikiname wikiname #:title prefixed-category #:body-class body-class + #:license license `(div ,(update-tree-wiki page wikiname) (hr) @@ -85,7 +88,8 @@ ("format" . "json"))))) (printf "out: ~a~n" dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)]) + (easy:response-json dest-res)] + [license (license-auto wikiname)]) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) (define page (html->xexp page-html)) @@ -93,15 +97,14 @@ (define body-class (match (regexp-match #rx"]*class=\"([^\"]*)" head-html) [(list _ classes) classes] [_ ""])) - (println head-html) - (println body-class) (define body (generate-results-page #:source-url source-url #:wikiname wikiname #:prefixed-category prefixed-category #:members-data members-data #:page page - #:body-class body-class)) + #:body-class body-class + #:license license)) (when (config-true? 'debug) ; used for its side effects @@ -117,5 +120,5 @@ #:source-url "" #:wikiname "test" #:prefixed-category "Category:Items" - #:category-data category-json-data + #:members-data category-json-data #:page '(div "page text")))))) diff --git a/src/page-search.rkt b/src/page-search.rkt index 5d1bd7df..61b1212a 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -10,8 +10,10 @@ web-server/http (only-in web-server/dispatchers/dispatch next-dispatcher) #;(only-in web-server/http/redirect redirect-to) - "config.rkt" "application-globals.rkt" + "config.rkt" + "data.rkt" + "syntax.rkt" "url-utils.rkt" "xexpr-utils.rkt") @@ -23,12 +25,13 @@ (define search-json-data '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) -(define (generate-results-page dest-url wikiname query data) +(define (generate-results-page dest-url wikiname query data #:license [license #f]) (define search-results (jp "/query/search" data)) (generate-wiki-page #:source-url dest-url #:wikiname wikiname #:title "Search Results" + #:license license `(div (@ (class "mw-parser-output")) (p ,(format "~a results found for " (length search-results)) (strong ,query)) @@ -54,29 +57,32 @@ (response-handler (define wikiname (path/param-path (first (url-path (request-uri req))))) (define query (dict-ref (url-query (request-uri req)) 'q #f)) - (define origin (format "https://~a.fandom.com" wikiname)) - (define dest-url (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "search") - ("srsearch" . ,query) - ("formatversion" . "2") - ("format" . "json"))))) - (printf "out: ~a~n" dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) + (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "query") + ("list" . "search") + ("srsearch" . ,query) + ("formatversion" . "2") + ("format" . "json"))))) - (define data (easy:response-json dest-res)) + (thread-let + ([dest-res (printf "out: ~a~n" dest-url) + (easy:get dest-url #:timeouts timeouts)] + [license (license-auto wikiname)]) - (define body (generate-results-page dest-url wikiname query data)) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - (λ (out) - (write-html body out))))) + (define data (easy:response-json dest-res)) + + (define body (generate-results-page dest-url wikiname query data #:license license)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + (λ (out) + (write-html body out)))))) (module+ test (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") (generate-results-page "" "test" "Gacha" search-json-data))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index ecfdbb35..2882ba5f 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -16,7 +16,9 @@ ; my libs "application-globals.rkt" "config.rkt" + "data.rkt" "pure-utils.rkt" + "syntax.rkt" "xexpr-utils.rkt" "url-utils.rkt") @@ -148,7 +150,7 @@ (λ (v) (and (eq? element-type 'a) (has-class? "image" v))) (λ (v) (dict-update v 'rel (λ (s) - (list (string-append (car s) " noreferrer"))) + (list (string-append (car s) " noreferrer"))) '("")))) ; proxy images from inline styles (curry attribute-maybe-update 'style @@ -235,49 +237,53 @@ (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) - (define dest-url (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,path) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (printf "out: ~a~n" dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) + (thread-let + ([dest-res (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "parse") + ("page" . ,path) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))))) + (printf "out: ~a~n" dest-url) + (easy:get dest-url #:timeouts timeouts)] + [license (license-auto wikiname)]) - (cond - [(eq? 200 (easy:response-status-code dest-res)) - (let* ([data (easy:response-json dest-res)] - [title (jp "/parse/title" data "")] - [page-html (jp "/parse/text" data "")] - [page-html (preprocess-html-wiki page-html)] - [page (html->xexp page-html)] - [head-html (jp "/parse/headhtml" data "")] - [body-class (match (regexp-match #rx"]*class=\"([^\"]*)" head-html) - [(list _ classes) classes] - [_ ""])]) - (if (equal? "missingtitle" (jp "/error/code" data #f)) - (next-dispatcher) - (response-handler - (define body - (generate-wiki-page - (update-tree-wiki page wikiname) - #:source-url source-url - #:wikiname wikiname - #:title title - #:body-class body-class)) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) - (define headers (if redirect-msg - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] - [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value))) - (list))) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - #:headers headers - (λ (out) - (write-html body out))))))])) + (cond + [(eq? 200 (easy:response-status-code dest-res)) + (let* ([data (easy:response-json dest-res)] + [title (jp "/parse/title" data "")] + [page-html (jp "/parse/text" data "")] + [page-html (preprocess-html-wiki page-html)] + [page (html->xexp page-html)] + [head-html (jp "/parse/headhtml" data "")] + [body-class (match (regexp-match #rx"]*class=\"([^\"]*)" head-html) + [(list _ classes) classes] + [_ ""])]) + (if (equal? "missingtitle" (jp "/error/code" data #f)) + (next-dispatcher) + (response-handler + (define body + (generate-wiki-page + (update-tree-wiki page wikiname) + #:source-url source-url + #:wikiname wikiname + #:title title + #:body-class body-class + #:license license)) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define headers (if redirect-msg + (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) + (list (header #"Refresh" value))) + (list))) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + #:headers headers + (λ (out) + (write-html body out))))))]))) From 82978e7c139e26b53b029e4f55a3afc7755e3a44 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 22:34:34 +1200 Subject: [PATCH 027/166] Remove table colour override This is no longer necessary because the table colours are okay because the page theme is now correctly set. --- static/main.css | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/static/main.css b/static/main.css index 7ef51a13..ba0eaadf 100644 --- a/static/main.css +++ b/static/main.css @@ -26,6 +26,9 @@ sup { sub { vertical-align: sub; } +.page table { + color: var(--theme-page-text-color); /* no idea why this needs to be specified, it should inherit from .page */ +} /* general page appearance */ body.skin-fandomdesktop, button, input, textarea, .wikitable, .va-table { @@ -58,6 +61,15 @@ p { margin-top: 0; } +/* home page search form */ +.paired__label { + display: grid; +} +.paired__input { + width: auto; + max-width: 240px; +} + /* custom footer with source and license info */ .custom-footer { clear: both; @@ -132,20 +144,6 @@ img { overflow: auto; } -/* table overrides for dark theme pages */ -/* these colours look good in all wikis I tried. colours that don't look good: - --theme-page-background-color--secondary is bad for fallout - --theme-page-accent-mix-color is bad for minecraft */ -.wikitable > tr > th, .wikitable > * > tr > th, .va-table th { - background-color: var(--theme-page-dynamic-color-1--inverted); -} -.wikitable td, .va-table td { - background-color: var(--theme-page-text-mix-color-95); -} -.page table { - color: var(--theme-page-text-color); /* no idea why this needs to be specified, it should inherit from .page */ -} - /* float right. reorganised from their sheet */ .tright { clear: right; From 711a8225fb04415f925805e5395c0c34c4307e15 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 22:36:04 +1200 Subject: [PATCH 028/166] Add global search on home page --- breezewiki.rkt | 2 ++ dist.rkt | 2 ++ src/dispatcher-tree.rkt | 1 + src/page-global-search.rkt | 25 +++++++++++++++++++++++++ src/page-home.rkt | 15 ++++++++++++--- 5 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 src/page-global-search.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index aeaaa890..dfb405e4 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -10,6 +10,7 @@ (make-reloadable-entry-point (quote varname) filename)))) (require-reloadable "src/page-category.rkt" page-category) +(require-reloadable "src/page-global-search.rkt" page-global-search) (require-reloadable "src/page-home.rkt" page-home) (require-reloadable "src/page-not-found.rkt" page-not-found) (require-reloadable "src/page-proxy.rkt" page-proxy) @@ -31,6 +32,7 @@ (dispatcher-tree ; order of these does not matter page-category + page-global-search page-home page-not-found page-proxy diff --git a/dist.rkt b/dist.rkt index 596a1b97..a6266951 100644 --- a/dist.rkt +++ b/dist.rkt @@ -4,6 +4,7 @@ "src/dispatcher-tree.rkt") (require (only-in "src/page-category.rkt" page-category)) +(require (only-in "src/page-global-search.rkt" page-global-search)) (require (only-in "src/page-home.rkt" page-home)) (require (only-in "src/page-not-found.rkt" page-not-found)) (require (only-in "src/page-proxy.rkt" page-proxy)) @@ -20,6 +21,7 @@ (dispatcher-tree ; order of these does not matter page-category + page-global-search page-home page-not-found page-proxy diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 5800876d..16f6c71d 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -42,6 +42,7 @@ (sequencer:make (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) + (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) diff --git a/src/page-global-search.rkt b/src/page-global-search.rkt new file mode 100644 index 00000000..4364dea0 --- /dev/null +++ b/src/page-global-search.rkt @@ -0,0 +1,25 @@ +#lang racket/base +(require racket/dict + ; web server libs + net/url + web-server/http + "application-globals.rkt" + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide + page-global-search) + +(define (page-global-search req) + (define wikiname (dict-ref (url-query (request-uri req)) 'wikiname #f)) + (define q (dict-ref (url-query (request-uri req)) 'q #f)) + (response-handler + (if (not (and wikiname q)) + (response/output + #:code 400 + #:mime-type "text/plain" + (λ (out) + (displayln "Requires wikiname and q parameters." out))) + (generate-redirect (format "/~a/search?~a" + wikiname + (params->query `(("q" . ,q)))))))) diff --git a/src/page-home.rkt b/src/page-home.rkt index 3c84511f..7c7aaa1d 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -27,11 +27,20 @@ (p "It removes ads, videos, and suggested content, leaving you with a clean page that doesn't slow down your device or use up your data.") (p "BreezeWiki can also be called an \"alternative frontend for Fandom\".") (p ,(format "To use BreezeWiki, just replace \"fandom.com\" with \"~a\", and you'll instantly be teleported to a better world." - (if (config-true? 'canonical_origin) - (url-host (string->url (config-get 'canonical_origin))) - "breezewiki.com"))) + (if (config-true? 'canonical_origin) + (url-host (string->url (config-get 'canonical_origin))) + "breezewiki.com"))) (p "If you'd like to be automatically sent to BreezeWiki every time in the future, " (a (@ (href "https://docs.breezewiki.com/Automatic_Redirection.html")) "check out the tutorial in the manual.")) + (h2 "Find a page") + (form (@ (action "/search")) + (label (@ (class "paired__label")) + "Wiki name" + (input (@ (name "wikiname") (class "paired__input") (type "text") (placeholder "pokemon") (required)))) + (label (@ (class "paired__label")) + "Search query" + (input (@ (name "q") (class "paired__input") (type "text") (placeholder "Eevee") (required)))) + (button "Search")) (h2 "Example pages") (ul ,@(map (λ (x) From b57fd99c7d39445ee69c6db530ebd289193a86c8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 22:51:42 +1200 Subject: [PATCH 029/166] Support environment variables for configuration --- src/config.rkt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index 16d1daff..b4c1cc14 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -1,6 +1,8 @@ #lang racket/base -(require racket/pretty +(require racket/function + racket/pretty racket/runtime-path + racket/string ini) (provide @@ -51,7 +53,16 @@ l (printf "note: ~a items loaded from config file~n" (length l))))) -(define combined-alist (append default-config loaded-alist)) +(define env-alist + (let ([e-names (environment-variables-names (current-environment-variables))] + [e-ref (λ (name) (bytes->string/latin-1 (environment-variables-ref (current-environment-variables) name)))]) + (map (λ (name) (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) + (e-ref name))) + (filter (λ (name) (string-prefix? (bytes->string/latin-1 name) "BW_")) e-names)))) +(when (> (length env-alist) 0) + (printf "note: ~a items loaded from environment variables~n" (length env-alist))) + +(define combined-alist (append default-config loaded-alist env-alist)) (define config (make-hasheq From 4d8746e85ee8867ee6ecc8daad4f849f108f6c1e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 22:55:25 +1200 Subject: [PATCH 030/166] Case insensitive environment variables --- src/config.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rkt b/src/config.rkt index b4c1cc14..a5ceba39 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -58,7 +58,7 @@ [e-ref (λ (name) (bytes->string/latin-1 (environment-variables-ref (current-environment-variables) name)))]) (map (λ (name) (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) (e-ref name))) - (filter (λ (name) (string-prefix? (bytes->string/latin-1 name) "BW_")) e-names)))) + (filter (λ (name) (string-prefix? (string-downcase (bytes->string/latin-1 name)) "bw_")) e-names)))) (when (> (length env-alist) 0) (printf "note: ~a items loaded from environment variables~n" (length env-alist))) From 3d8feaba9a0145fd3ba9dc4f14b5f23e13b76ee8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 17 Sep 2022 23:09:47 +1200 Subject: [PATCH 031/166] Warn if canonical_origin not present in production --- src/config.rkt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config.rkt b/src/config.rkt index a5ceba39..bdf44401 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -78,3 +78,9 @@ (hash->list (make-hasheq combined-alist)) symbol Date: Sun, 25 Sep 2022 16:45:07 +0700 Subject: [PATCH 032/166] Display real category title Fixes https://lists.sr.ht/~cadence/breezewiki-discuss/%3CCN5B6AUHFUSC.1H7B7SEFDRDK9%40archer%3E --- src/page-category.rkt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/page-category.rkt b/src/page-category.rkt index 8ecd3689..f9eb9740 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -31,7 +31,7 @@ (define (generate-results-page #:source-url source-url #:wikiname wikiname - #:prefixed-category prefixed-category + #:title title #:members-data members-data #:page page #:body-class [body-class #f] @@ -40,13 +40,13 @@ (generate-wiki-page #:source-url source-url #:wikiname wikiname - #:title prefixed-category + #:title title #:body-class body-class #:license license `(div ,(update-tree-wiki page wikiname) (hr) - (h2 ,(format "All Pages in ~a" prefixed-category)) + (h2 ,(format "All Pages in ~a" title)) (div (@ (class "mw-parser-output")) (ul (@ (class "my-category-list")) ,@(map @@ -91,6 +91,7 @@ (easy:response-json dest-res)] [license (license-auto wikiname)]) + (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) (define page (html->xexp page-html)) (define head-html (jp "/parse/headhtml" page-data "")) @@ -100,7 +101,7 @@ (define body (generate-results-page #:source-url source-url #:wikiname wikiname - #:prefixed-category prefixed-category + #:title title #:members-data members-data #:page page #:body-class body-class @@ -119,6 +120,6 @@ (generate-results-page #:source-url "" #:wikiname "test" - #:prefixed-category "Category:Items" + #:title "Category:Items" #:members-data category-json-data #:page '(div "page text")))))) From 2a56107e979a5a2f5a16c61f0c631000aad8ce54 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 2 Oct 2022 09:59:19 +0700 Subject: [PATCH 033/166] Fix wikis with numbers in the name https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/extensions/wikia/CreateNewWiki/CreateWikiChecks.php#L112 Fixes https://lists.sr.ht/~cadence/breezewiki-discuss/%3C814ef2a8-2e91-dcaf-f0c1-805cc1478198%40riseup.net%3E --- src/dispatcher-tree.rkt | 8 ++++---- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 16f6c71d..93532ef1 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -43,9 +43,9 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (filter:make #rx"^/[a-z-]+/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) - (filter:make #rx"^/[a-z-]+/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) - (filter:make #rx"^/[a-z-]+/search$" (lift:make (hash-ref ds 'page-search))) - (filter:make #rx"^/[a-z-]+(/(wiki(/)?)?)?$" (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) + (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) + (filter:make #px"^/[a-zA-Z0-9-]{3,50}/search$" (lift:make (hash-ref ds 'page-search))) + (filter:make #px"^/[a-zA-Z0-9-]{3,50}(/(wiki(/)?)?)?$" (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 2882ba5f..96410d02 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -143,7 +143,7 @@ (λ (href) ((compose1 (λ (href) (regexp-replace #rx"^(/wiki/.*)" href (format "/~a\\1" wikiname))) - (λ (href) (regexp-replace #rx"^https://([a-z-]+).fandom.com(/wiki/.*)" href "/\\1\\2"))) + (λ (href) (regexp-replace #px"^https://([a-zA-Z0-9-]{3,50}).fandom.com(/wiki/.*)" href "/\\1\\2"))) href))) ; add noreferrer to a.image (curry u diff --git a/src/url-utils.rkt b/src/url-utils.rkt index f563b69a..934a228b 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -57,7 +57,7 @@ (: is-fandom-url? (String -> Boolean)) (define (is-fandom-url? url) - (regexp-match? #rx"^https://static.wikia.nocookie.net/|^https://[a-z-]*.fandom.com/" url)) + (regexp-match? #px"^https://static.wikia.nocookie.net/|^https://[a-zA-Z0-9-]{3,50}.fandom.com/" url)) (module+ test (check-true (is-fandom-url? "https://static.wikia.nocookie.net/wikiname/images/2/2f/SomeImage.jpg/revision/latest?cb=20110210094136")) (check-true (is-fandom-url? "https://test.fandom.com/wiki/Some_Page")) From 10cdd260e0a6bec81c8be0a261851661d5a30059 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:00:33 +1300 Subject: [PATCH 034/166] Close response after error in proxy --- src/page-proxy.rkt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/page-proxy.rkt b/src/page-proxy.rkt index f494e283..8dbf0f91 100644 --- a/src/page-proxy.rkt +++ b/src/page-proxy.rkt @@ -18,13 +18,16 @@ (match (dict-ref (url-query (request-uri req)) 'dest #f) [(? string? dest) (if (is-fandom-url? dest) - (response-handler + (response-handler ; catches and reports errors (let ([dest-r (easy:get dest #:stream? #t)]) - (response/output - #:code (easy:response-status-code dest-r) - #:mime-type (easy:response-headers-ref dest-r 'content-type) - (λ (out) - (copy-port (easy:response-output dest-r) out) - (easy:response-close! dest-r))))) + (with-handlers ([exn:fail? (λ (e) ; cleans up and re-throws + (easy:response-close! dest-r) + (raise e))]) + (response/output + #:code (easy:response-status-code dest-r) + #:mime-type (easy:response-headers-ref dest-r 'content-type) + (λ (out) + (copy-port (easy:response-output dest-r) out) + (easy:response-close! dest-r)))))) (next-dispatcher))] [#f (next-dispatcher)])) From ece762fc5b9b4632674cc1b86c50a3a1ecca4423 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:13:07 +1300 Subject: [PATCH 035/166] Unify "out: " logging to a function --- src/config.rkt | 1 + src/data.rkt | 2 +- src/page-category.rkt | 4 ++-- src/page-search.rkt | 2 +- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 10 +++++++++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index bdf44401..4c8fca97 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -26,6 +26,7 @@ (canonical_origin . "") (debug . "false") (instance_is_official . "false") ; please don't turn this on, or you will make me very upset + (log_outgoing . "true") (port . "10416") (strict_proxy . "true"))) diff --git a/src/data.rkt b/src/data.rkt index f7a0d1f4..8eb0cd69 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -20,7 +20,7 @@ ("siprop" . "rightsinfo") ("format" . "json") ("formatversion" . "2"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define res (easy:get dest-url)) (define data (easy:response-json res)) (license (jp "/query/rightsinfo/text" data) diff --git a/src/page-category.rkt b/src/page-category.rkt index f9eb9740..f7c43b27 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -75,7 +75,7 @@ ("cmlimit" . "max") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] [page-data (define dest-url @@ -86,7 +86,7 @@ ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] [license (license-auto wikiname)]) diff --git a/src/page-search.rkt b/src/page-search.rkt index 61b1212a..387deabc 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -68,7 +68,7 @@ ("format" . "json"))))) (thread-let - ([dest-res (printf "out: ~a~n" dest-url) + ([dest-res (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] [license (license-auto wikiname)]) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 96410d02..461594ac 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -246,7 +246,7 @@ ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json"))))) - (printf "out: ~a~n" dest-url) + (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] [license (license-auto wikiname)]) diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 934a228b..20b9b421 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -1,6 +1,7 @@ #lang typed/racket/base (require racket/string "pure-utils.rkt") +(require/typed "config.rkt" [config-true? (Symbol -> Boolean)]) (provide ; make a query string from an association list of strings @@ -8,7 +9,9 @@ ; make a proxied version of a fandom url u-proxy-url ; check whether a url is on a domain controlled by fandom - is-fandom-url?) + is-fandom-url? + ; prints "out: " + log-outgoing) (module+ test (require "typed-rackunit.rkt")) @@ -69,3 +72,8 @@ is-fandom-url? (λ ([v : String]) (string-append "/proxy?" (params->query `(("dest" . ,url))))) url)) + +(: log-outgoing (String -> Void)) +(define (log-outgoing url-string) + (when (config-true? 'log_outgoing) + (printf "out: ~a~n" url-string))) From 57e700cef547da41e95884a33ca2153a8f48f461 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 2 Oct 2022 16:44:44 +0700 Subject: [PATCH 036/166] Deduplicate the wiki name regex --- src/dispatcher-tree.rkt | 11 ++++++----- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 6 +++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 93532ef1..1a434588 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -8,7 +8,8 @@ (prefix-in sequencer: web-server/dispatchers/dispatch-sequencer) (prefix-in lift: web-server/dispatchers/dispatch-lift) (prefix-in filter: web-server/dispatchers/dispatch-filter) - "config.rkt") + "config.rkt" + "url-utils.rkt") (provide ; syntax to make the hashmap from names @@ -43,9 +44,9 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/Category:.+$" (lift:make (hash-ref ds 'page-category))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/wiki/.+$" (lift:make (hash-ref ds 'page-wiki))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}/search$" (lift:make (hash-ref ds 'page-search))) - (filter:make #px"^/[a-zA-Z0-9-]{3,50}(/(wiki(/)?)?)?$" (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/search$" wikiname-regex)) (lift:make (hash-ref ds 'page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" wikiname-regex)) (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 461594ac..4d6fc186 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -143,7 +143,7 @@ (λ (href) ((compose1 (λ (href) (regexp-replace #rx"^(/wiki/.*)" href (format "/~a\\1" wikiname))) - (λ (href) (regexp-replace #px"^https://([a-zA-Z0-9-]{3,50}).fandom.com(/wiki/.*)" href "/\\1\\2"))) + (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" wikiname-regex)) href "/\\1\\2"))) href))) ; add noreferrer to a.image (curry u diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 20b9b421..07133e70 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -4,6 +4,8 @@ (require/typed "config.rkt" [config-true? (Symbol -> Boolean)]) (provide + ; regex to match wiki names + wikiname-regex ; make a query string from an association list of strings params->query ; make a proxied version of a fandom url @@ -16,6 +18,8 @@ (module+ test (require "typed-rackunit.rkt")) +(define wikiname-regex "[a-zA-Z0-9-]{3,50}") + ;; https://url.spec.whatwg.org/#urlencoded-serializing (define urlencoded-set '(#\! #\' #\( #\) #\~ ; urlencoded set @@ -60,7 +64,7 @@ (: is-fandom-url? (String -> Boolean)) (define (is-fandom-url? url) - (regexp-match? #px"^https://static.wikia.nocookie.net/|^https://[a-zA-Z0-9-]{3,50}.fandom.com/" url)) + (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" wikiname-regex)) url)) (module+ test (check-true (is-fandom-url? "https://static.wikia.nocookie.net/wikiname/images/2/2f/SomeImage.jpg/revision/latest?cb=20110210094136")) (check-true (is-fandom-url? "https://test.fandom.com/wiki/Some_Page")) From 79f04565c72acc683b4e6a153d5623adb2bf25da Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 21:18:30 +1300 Subject: [PATCH 037/166] Rename wikiname-regex to px-wikiname I find prefix notation more natural to represent the type/kind of the thing. --- src/dispatcher-tree.rkt | 8 ++++---- src/page-wiki.rkt | 2 +- src/url-utils.rkt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 1a434588..f2c1412d 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -44,9 +44,9 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (filter:make (pregexp (format "^/~a/wiki/Category:.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-category))) - (filter:make (pregexp (format "^/~a/wiki/.+$" wikiname-regex)) (lift:make (hash-ref ds 'page-wiki))) - (filter:make (pregexp (format "^/~a/search$" wikiname-regex)) (lift:make (hash-ref ds 'page-search))) - (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" wikiname-regex)) (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found))))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 4d6fc186..a218dfeb 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -143,7 +143,7 @@ (λ (href) ((compose1 (λ (href) (regexp-replace #rx"^(/wiki/.*)" href (format "/~a\\1" wikiname))) - (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" wikiname-regex)) href "/\\1\\2"))) + (λ (href) (regexp-replace (pregexp (format "^https://(~a)\\.fandom\\.com(/wiki/.*)" px-wikiname)) href "/\\1\\2"))) href))) ; add noreferrer to a.image (curry u diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 07133e70..a55a208b 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -5,7 +5,7 @@ (provide ; regex to match wiki names - wikiname-regex + px-wikiname ; make a query string from an association list of strings params->query ; make a proxied version of a fandom url @@ -18,7 +18,7 @@ (module+ test (require "typed-rackunit.rkt")) -(define wikiname-regex "[a-zA-Z0-9-]{3,50}") +(define px-wikiname "[a-zA-Z0-9-]{3,50}") ;; https://url.spec.whatwg.org/#urlencoded-serializing @@ -64,7 +64,7 @@ (: is-fandom-url? (String -> Boolean)) (define (is-fandom-url? url) - (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" wikiname-regex)) url)) + (regexp-match? (pregexp (format "^https://static\\.wikia\\.nocookie\\.net/|^https://~a\\.fandom\\.com/" px-wikiname)) url)) (module+ test (check-true (is-fandom-url? "https://static.wikia.nocookie.net/wikiname/images/2/2f/SomeImage.jpg/revision/latest?cb=20110210094136")) (check-true (is-fandom-url? "https://test.fandom.com/wiki/Some_Page")) From 6b176e3f8f58152620e0a18c7fa57e3fb229a8bb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Oct 2022 22:00:44 +1300 Subject: [PATCH 038/166] Migrate config.rkt to Typed Racket --- src/config.rkt | 44 +++++++++++++++++++++++++++++++++----------- src/url-utils.rkt | 2 +- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/config.rkt b/src/config.rkt index 4c8fca97..6fc029ff 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -1,23 +1,32 @@ -#lang racket/base +#lang typed/racket/base (require racket/function racket/pretty racket/runtime-path - racket/string - ini) + racket/string) +(require/typed ini + [#:opaque Ini ini?] + [read-ini (Input-Port -> Ini)] + [ini->hash (Ini -> (Immutable-HashTable Symbol (Immutable-HashTable Symbol String)))]) (provide config-parameter config-true? config-get) +(module+ test + (require "typed-rackunit.rkt")) + (define-runtime-path path-config "../config.ini") +(: config-parameter (Symbol -> (Parameterof String))) (define (config-parameter key) (hash-ref config key)) +(: config-true? (Symbol -> Boolean)) (define (config-true? key) (not (member ((config-parameter key)) '("" "false")))) +(: config-get (Symbol -> String)) (define (config-get key) ((config-parameter key))) @@ -56,18 +65,24 @@ (define env-alist (let ([e-names (environment-variables-names (current-environment-variables))] - [e-ref (λ (name) (bytes->string/latin-1 (environment-variables-ref (current-environment-variables) name)))]) - (map (λ (name) (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) - (e-ref name))) - (filter (λ (name) (string-prefix? (string-downcase (bytes->string/latin-1 name)) "bw_")) e-names)))) + [e-ref (λ ([name : Bytes]) + (bytes->string/latin-1 + (cast (environment-variables-ref (current-environment-variables) name) + Bytes)))]) + (map (λ ([name : Bytes]) + (cons (string->symbol (string-downcase (substring (bytes->string/latin-1 name) 3))) + (e-ref name))) + (filter (λ ([name : Bytes]) (string-prefix? (string-downcase (bytes->string/latin-1 name)) + "bw_")) + e-names)))) (when (> (length env-alist) 0) (printf "note: ~a items loaded from environment variables~n" (length env-alist))) (define combined-alist (append default-config loaded-alist env-alist)) (define config - (make-hasheq - (map (λ (pair) + (make-immutable-hasheq + (map (λ ([pair : (Pairof Symbol String)]) (cons (car pair) (make-parameter (cdr pair)))) combined-alist))) @@ -75,8 +90,8 @@ ; all values here are optimised for maximum prettiness (parameterize ([pretty-print-columns 80]) (display "config: ") - (pretty-write (sort - (hash->list (make-hasheq combined-alist)) + (pretty-write ((inst sort (Pairof Symbol String)) + (hash->list (make-immutable-hasheq combined-alist)) symbol Boolean)]) (provide ; regex to match wiki names From 9aba3ad4320229dc9837a0fa647f1f27f83bbf58 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Oct 2022 20:53:09 +1300 Subject: [PATCH 039/166] Refactor siteinfo/license fetching --- info.rkt | 2 +- src/data.rkt | 29 +++++++++++++++-------------- src/page-category.rkt | 2 +- src/page-search.rkt | 2 +- src/page-wiki.rkt | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/info.rkt b/info.rkt index 74152ef9..46512dfd 100644 --- a/info.rkt +++ b/info.rkt @@ -1,3 +1,3 @@ #lang info -(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib")) +(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib" "memo")) diff --git a/src/data.rkt b/src/data.rkt index 8eb0cd69..fae07d3e 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -1,33 +1,34 @@ #lang racket/base -(require (prefix-in easy: net/http-easy) +(require racket/list + (prefix-in easy: net/http-easy) + memo "url-utils.rkt" "xexpr-utils.rkt") (provide + (struct-out siteinfo) (struct-out license) - license-default - license-auto) + siteinfo-fetch + license-default) +(struct siteinfo (sitename basepage license) #:transparent) (struct license (text url) #:transparent) + (define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing")) -(define license-hash (make-hash)) -(define (license-fetch wikiname) + +(define/memoize (siteinfo-fetch wikiname) #:hash hash (define dest-url (format "https://~a.fandom.com/api.php?~a" wikiname (params->query '(("action" . "query") ("meta" . "siteinfo") - ("siprop" . "rightsinfo") + ("siprop" . "general|rightsinfo") ("format" . "json") ("formatversion" . "2"))))) (log-outgoing dest-url) (define res (easy:get dest-url)) (define data (easy:response-json res)) - (license (jp "/query/rightsinfo/text" data) - (jp "/query/rightsinfo/url" data))) -(define (license-auto wikiname) - (if (hash-has-key? license-hash wikiname) - (hash-ref license-hash wikiname) - (let ([result (license-fetch wikiname)]) - (hash-set! license-hash wikiname result) - result))) + (siteinfo (jp "/query/general/sitename" data) + (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) + (license (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data)))) diff --git a/src/page-category.rkt b/src/page-category.rkt index f7c43b27..773985db 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -89,7 +89,7 @@ (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] - [license (license-auto wikiname)]) + [license (siteinfo-license (siteinfo-fetch wikiname))]) (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) diff --git a/src/page-search.rkt b/src/page-search.rkt index 387deabc..496dac81 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -70,7 +70,7 @@ (thread-let ([dest-res (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] - [license (license-auto wikiname)]) + [license (siteinfo-license (siteinfo-fetch wikiname))]) (define data (easy:response-json dest-res)) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index a218dfeb..32255e1b 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -248,7 +248,7 @@ ("format" . "json"))))) (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] - [license (license-auto wikiname)]) + [license (siteinfo-license (siteinfo-fetch wikiname))]) (cond [(eq? 200 (easy:response-status-code dest-res)) From 59332fd9d13e7c66523816b0a96d9ab8c1728559 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Oct 2022 22:50:50 +1300 Subject: [PATCH 040/166] Pass siteinfo through code; show sitename in title --- src/application-globals.rkt | 14 +++++++++----- src/data.rkt | 20 +++++++++++--------- src/page-category.rkt | 8 ++++---- src/page-search.rkt | 8 ++++---- src/page-wiki.rkt | 4 ++-- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index c2149246..6b5f3d9e 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -48,8 +48,8 @@ ,(if source-url `(div (p "This page displays proxied content from " (a (@ (href ,source-url) (rel "noreferrer")) ,source-url) - ,(format ". Text content is available under the ~a license, " (license-text license)) - (a (@ (href ,(license-url license))) "see license info.") + ,(format ". Text content is available under the ~a license, " (license^-text license)) + (a (@ (href ,(license^-url license))) "see license info.") " Media files may have different copying restrictions.") (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))) `(div (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " @@ -63,7 +63,8 @@ #:wikiname wikiname #:title title #:body-class [body-class-in #f] - #:license [license #f]) + #:siteinfo [siteinfo-in #f]) + (define siteinfo (or siteinfo-in siteinfo-default)) (define body-class (if (not body-class-in) "skin-fandomdesktop" body-class-in)) @@ -82,7 +83,10 @@ `(html (head (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) - (title ,(format "~a | ~a" title (config-get 'application_name))) + (title ,(format "~a | ~a+~a" + title + (regexp-replace #rx" ?Wiki$" (siteinfo^-sitename siteinfo) "") + (config-get 'application_name))) ,@(map (λ (url) `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) (required-styles (format "https://~a.fandom.com" wikiname))) @@ -101,7 +105,7 @@ (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,content)) - ,(application-footer source-url #:license license))))))) + ,(application-footer source-url #:license (siteinfo^-license siteinfo)))))))) (module+ test (define page (parameterize ([(config-parameter 'strict_proxy) "true"]) diff --git a/src/data.rkt b/src/data.rkt index fae07d3e..6673e4c9 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -6,15 +6,17 @@ "xexpr-utils.rkt") (provide - (struct-out siteinfo) - (struct-out license) + (struct-out siteinfo^) + (struct-out license^) siteinfo-fetch + siteinfo-default license-default) -(struct siteinfo (sitename basepage license) #:transparent) -(struct license (text url) #:transparent) +(struct siteinfo^ (sitename basepage license) #:transparent) +(struct license^ (text url) #:transparent) -(define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing")) +(define license-default (license^ "CC-BY-SA" "https://www.fandom.com/licensing")) +(define siteinfo-default (siteinfo^ "Test Wiki" "Main_Page" license-default)) (define/memoize (siteinfo-fetch wikiname) #:hash hash (define dest-url @@ -28,7 +30,7 @@ (log-outgoing dest-url) (define res (easy:get dest-url)) (define data (easy:response-json res)) - (siteinfo (jp "/query/general/sitename" data) - (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) - (license (jp "/query/rightsinfo/text" data) - (jp "/query/rightsinfo/url" data)))) + (siteinfo^ (jp "/query/general/sitename" data) + (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) + (license^ (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data)))) diff --git a/src/page-category.rkt b/src/page-category.rkt index 773985db..bf8b982c 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -35,14 +35,14 @@ #:members-data members-data #:page page #:body-class [body-class #f] - #:license [license #f]) + #:siteinfo [siteinfo #f]) (define members (jp "/query/categorymembers" members-data)) (generate-wiki-page #:source-url source-url #:wikiname wikiname #:title title #:body-class body-class - #:license license + #:siteinfo siteinfo `(div ,(update-tree-wiki page wikiname) (hr) @@ -89,7 +89,7 @@ (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] - [license (siteinfo-license (siteinfo-fetch wikiname))]) + [siteinfo (siteinfo-fetch wikiname)]) (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) @@ -105,7 +105,7 @@ #:members-data members-data #:page page #:body-class body-class - #:license license)) + #:siteinfo siteinfo)) (when (config-true? 'debug) ; used for its side effects diff --git a/src/page-search.rkt b/src/page-search.rkt index 496dac81..0647e575 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -25,13 +25,13 @@ (define search-json-data '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) -(define (generate-results-page dest-url wikiname query data #:license [license #f]) +(define (generate-results-page dest-url wikiname query data #:siteinfo [siteinfo #f]) (define search-results (jp "/query/search" data)) (generate-wiki-page #:source-url dest-url #:wikiname wikiname #:title "Search Results" - #:license license + #:siteinfo siteinfo `(div (@ (class "mw-parser-output")) (p ,(format "~a results found for " (length search-results)) (strong ,query)) @@ -70,11 +70,11 @@ (thread-let ([dest-res (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] - [license (siteinfo-license (siteinfo-fetch wikiname))]) + [siteinfo (siteinfo-fetch wikiname)]) (define data (easy:response-json dest-res)) - (define body (generate-results-page dest-url wikiname query data #:license license)) + (define body (generate-results-page dest-url wikiname query data #:siteinfo siteinfo)) (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 32255e1b..41dc215c 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -248,7 +248,7 @@ ("format" . "json"))))) (log-outgoing dest-url) (easy:get dest-url #:timeouts timeouts)] - [license (siteinfo-license (siteinfo-fetch wikiname))]) + [siteinfo (siteinfo-fetch wikiname)]) (cond [(eq? 200 (easy:response-status-code dest-res)) @@ -271,7 +271,7 @@ #:wikiname wikiname #:title title #:body-class body-class - #:license license)) + #:siteinfo siteinfo)) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] From ade7878f7b090cfef9860d4e98c2486d9edbbfb8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Oct 2022 22:54:59 +1300 Subject: [PATCH 041/166] Redirect to actual wiki main page --- src/page-redirect-wiki-home.rkt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/page-redirect-wiki-home.rkt b/src/page-redirect-wiki-home.rkt index b0c3df3c..c8e6dde9 100644 --- a/src/page-redirect-wiki-home.rkt +++ b/src/page-redirect-wiki-home.rkt @@ -2,6 +2,7 @@ (require net/url web-server/http "application-globals.rkt" + "data.rkt" "url-utils.rkt" "xexpr-utils.rkt") @@ -11,5 +12,6 @@ (define (redirect-wiki-home req) (response-handler (define wikiname (path/param-path (car (url-path (request-uri req))))) - (define dest (format "/~a/wiki/Main_Page" wikiname)) + (define siteinfo (siteinfo-fetch wikiname)) + (define dest (format "/~a/wiki/~a" wikiname (or (siteinfo^-basepage siteinfo) "Main_Page"))) (generate-redirect dest))) From adc4b47b83e7e850c242b6be854060f0908b70e4 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:53:02 +0700 Subject: [PATCH 042/166] Set Referrer-Policy to no-referrer Fandom sends a fake 404 to media if there's a Referer header that has an origin that's not Fandom. However, we can choose not to send the header by setting Referrer-Policy. See also: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy --- src/application-globals.rkt | 3 +++ src/page-category.rkt | 1 + src/page-search.rkt | 1 + src/page-wiki.rkt | 30 +++++++++++++++++------------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 6b5f3d9e..3230b58a 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -9,6 +9,8 @@ "url-utils.rkt") (provide + ; header to not send referers to fandom + referrer-policy ; timeout durations for http-easy requests timeouts ; generates a consistent footer @@ -22,6 +24,7 @@ (require rackunit html-writing)) +(define referrer-policy (header #"Referrer-Policy" #"no-referrer")) (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define (application-footer source-url #:license [license-in #f]) diff --git a/src/page-category.rkt b/src/page-category.rkt index bf8b982c..036b59c1 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -113,6 +113,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-search.rkt b/src/page-search.rkt index 0647e575..e951749f 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -81,6 +81,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 41dc215c..b0396955 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -152,15 +152,17 @@ (λ (v) (dict-update v 'rel (λ (s) (list (string-append (car s) " noreferrer"))) '("")))) - ; proxy images from inline styles - (curry attribute-maybe-update 'style - (λ (style) - (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style - (λ (whole url) - (string-append - "url(" - (u-proxy-url url) - ")"))))) + ; proxy images from inline styles, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'style + (λ (style) + (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style + (λ (whole url) + (string-append + "url(" + (u-proxy-url url) + ")")))) v))) ; and also their links, if strict_proxy is set (curry u (λ (v) @@ -168,8 +170,10 @@ (eq? element-type 'a) (has-class? "image-thumbnail" v))) (λ (v) (attribute-maybe-update 'href u-proxy-url v))) - ; proxy images from src attributes - (curry attribute-maybe-update 'src u-proxy-url) + ; proxy images from src attributes, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'src u-proxy-url v))) ; don't lazyload images (curry u (λ (v) (dict-has-key? v 'data-src)) @@ -276,8 +280,8 @@ (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value))) - (list))) + (list (header #"Refresh" value) referrer-policy)) + (list referrer-policy))) (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid From 5813c49261c823503f712f07e40ee91e3f7a0b66 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 9 Oct 2022 23:43:21 +1300 Subject: [PATCH 043/166] Refactor Referrer-Policy header - Change the variable to always-headers so it can be extended in the future - New function build-headers that assists combining complex logic headers together with less mess - Also apply headers to the proxy --- src/application-globals.rkt | 7 ++++--- src/page-category.rkt | 2 +- src/page-home.rkt | 2 ++ src/page-proxy.rkt | 2 ++ src/page-search.rkt | 2 +- src/page-wiki.rkt | 12 +++++++----- src/url-utils.rkt | 19 ++++++++++++++++++- 7 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 3230b58a..7f16bee7 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -9,8 +9,8 @@ "url-utils.rkt") (provide - ; header to not send referers to fandom - referrer-policy + ; headers to always send on all http responses + always-headers ; timeout durations for http-easy requests timeouts ; generates a consistent footer @@ -24,7 +24,8 @@ (require rackunit html-writing)) -(define referrer-policy (header #"Referrer-Policy" #"no-referrer")) +(define always-headers + (list (header #"Referrer-Policy" #"same-origin"))) ; header to not send referers to fandom (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define (application-footer source-url #:license [license-in #f]) diff --git a/src/page-category.rkt b/src/page-category.rkt index 036b59c1..6c0a7337 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -113,7 +113,7 @@ (xexp->html body)) (response/output #:code 200 - #:headers (list referrer-policy) + #:headers (build-headers always-headers) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-home.rkt b/src/page-home.rkt index 7c7aaa1d..b16f66a1 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -4,6 +4,7 @@ html-writing web-server/http "application-globals.rkt" + "url-utils.rkt" "xexpr-utils.rkt" "config.rkt") @@ -81,6 +82,7 @@ (define (page-home req) (response/output #:code 200 + #:headers (build-headers always-headers) (λ (out) (write-html body out)))) diff --git a/src/page-proxy.rkt b/src/page-proxy.rkt index 8dbf0f91..3c22e1e3 100644 --- a/src/page-proxy.rkt +++ b/src/page-proxy.rkt @@ -8,6 +8,7 @@ net/url web-server/http (only-in web-server/dispatchers/dispatch next-dispatcher) + "application-globals.rkt" "url-utils.rkt" "xexpr-utils.rkt") @@ -26,6 +27,7 @@ (response/output #:code (easy:response-status-code dest-r) #:mime-type (easy:response-headers-ref dest-r 'content-type) + #:headers (build-headers always-headers) (λ (out) (copy-port (easy:response-output dest-r) out) (easy:response-close! dest-r)))))) diff --git a/src/page-search.rkt b/src/page-search.rkt index e951749f..81a88b2d 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -81,7 +81,7 @@ (xexp->html body)) (response/output #:code 200 - #:headers (list referrer-policy) + #:headers (build-headers always-headers) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index b0396955..effa40a0 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -277,11 +277,13 @@ #:body-class body-class #:siteinfo siteinfo)) (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) - (define headers (if redirect-msg - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] - [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value) referrer-policy)) - (list referrer-policy))) + (define headers + (build-headers + always-headers + (when redirect-msg + (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) + (header #"Refresh" value))))) (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 62f7cc2f..10df0891 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -2,6 +2,8 @@ (require racket/string "config.rkt" "pure-utils.rkt") +(require/typed web-server/http/request-structs + [#:opaque Header header?]) (provide ; regex to match wiki names @@ -13,7 +15,9 @@ ; check whether a url is on a domain controlled by fandom is-fandom-url? ; prints "out: " - log-outgoing) + log-outgoing + ; pass in a header, headers, or something useless. they'll all combine into a list + build-headers) (module+ test (require "typed-rackunit.rkt")) @@ -81,3 +85,16 @@ (define (log-outgoing url-string) (when (config-true? 'log_outgoing) (printf "out: ~a~n" url-string))) + +(: build-headers ((U Header (Listof Header) False Void) * -> (Listof Header))) +(define (build-headers . fs) + (apply + append + (map (λ ([f : (U Header (Listof Header) False Void)]) + (cond + [(not f) null] + [(void? f) null] + [(null? f) null] + [(header? f) (list f)] + [(pair? f) f])) + fs))) From 07074dccfc206d4fe3e489e536d3620ff7379370 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:35:35 +0700 Subject: [PATCH 044/166] Add support for File: pages Fixes https://lists.sr.ht/~cadence/breezewiki-discuss/%3Cb2835a70-5118-4df0-90c9-4333486a4b69%40nixnetmail.com%3E --- breezewiki.rkt | 2 + dist.rkt | 2 + src/dispatcher-tree.rkt | 1 + src/page-file.rkt | 166 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 src/page-file.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index dfb405e4..3fc9b8fb 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -19,6 +19,7 @@ (require-reloadable "src/page-static.rkt" static-dispatcher) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) +(require-reloadable "src/page-file.rkt" page-file) (reload!) @@ -38,6 +39,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher)))) diff --git a/dist.rkt b/dist.rkt index a6266951..805df48a 100644 --- a/dist.rkt +++ b/dist.rkt @@ -13,6 +13,7 @@ (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) (require (only-in "src/page-wiki.rkt" page-wiki)) +(require (only-in "src/page-file.rkt" page-file)) (serve/launch/wait #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f) @@ -27,6 +28,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index f2c1412d..b68cf9c7 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -45,6 +45,7 @@ (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) diff --git a/src/page-file.rkt b/src/page-file.rkt new file mode 100644 index 00000000..5c3f8965 --- /dev/null +++ b/src/page-file.rkt @@ -0,0 +1,166 @@ +#lang racket/base +(require racket/dict + racket/list + racket/match + racket/string + (prefix-in easy: net/http-easy) + ; html libs + html-parsing + html-writing + ; web server libs + net/url + web-server/http + (only-in web-server/dispatchers/dispatch next-dispatcher) + #;(only-in web-server/http/redirect redirect-to) + "application-globals.rkt" + "config.rkt" + "data.rkt" + "page-wiki.rkt" + "syntax.rkt" + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide page-file) + +(module+ test + (require rackunit) + (define test-media-detail + '#hasheq((fileTitle . "Example file") + (videoEmbedCode . "") + (imageUrl . "https://static.wikia.nocookie.net/examplefile") + (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") + (userName . "blankie") + (isPostedIn . #t) + (smallerArticleList . (#hasheq((title . "Example_article") + (titleText . "Example article")))) + (articleListIsSmaller . 0) + (exists . #t) + (imageDescription . #f)))) + +(define (url-content-type url) + (log-outgoing url) + (define dest-res (easy:head url #:timeouts timeouts)) + (easy:response-headers-ref dest-res 'content-type)) + +(define (get-media-html url content-type) + (define maybe-proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) + (cond + [(eq? content-type #f) `""] + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,maybe-proxied-url)))] + [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) + `(audio (@ (src ,maybe-proxied-url) (controls)))] + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,maybe-proxied-url) (controls)))] + [else `""])) + +(define (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:siteinfo [siteinfo #f]) + (define video-embed-code (jp "/videoEmbedCode" media-detail "")) + (define raw-image-url (jp "/rawImageUrl" media-detail)) + (define image-url (jp "/imageUrl" media-detail raw-image-url)) + (define username (jp "/userName" media-detail)) + (define is-posted-in (jp "/isPostedIn" media-detail #f)) + (define smaller-article-list (jp "/smallerArticleList" media-detail)) + (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) + (define image-description (jp "/imageDescription" media-detail #f)) + (define maybe-proxied-raw-image-url + (if (config-true? 'strict_proxy) (u-proxy-url raw-image-url) raw-image-url)) + (generate-wiki-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:siteinfo siteinfo + `(div ,(if (non-empty-string? video-embed-code) + (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) + (get-media-html image-url image-content-type)) + (p ,(if (non-empty-string? video-embed-code) + `"" + `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) + "Added by " + (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) + "." + ,(if is-posted-in + `(span " Posted in " + ,@(map (λ (article) + (define page-path (jp "/title" article)) + (define title (jp "/titleText" article page-path)) + `(span ,(if (eq? (car smaller-article-list) article) "" ", ") + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title))) + smaller-article-list) + ,(if (eq? article-list-is-smaller 1) "…" ".")) + `"")) + ,(if (string? image-description) + (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) + "")))) + +(define (page-file req) + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-title)) + + (thread-let ([media-detail + (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [siteinfo (siteinfo-fetch wikiname)]) + (if (not (jp "/exists" media-detail #f)) + (next-dispatcher) + (response-handler + (define file-title (jp "/fileTitle" media-detail "")) + (define title + (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) + (define image-content-type + (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) + #f + (url-content-type (jp "/imageUrl" media-detail)))) + (define body + (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:siteinfo siteinfo)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output #:code 200 + #:headers (build-headers always-headers) + (λ (out) (write-html body out))))))) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") + `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") + (controls))))) + (parameterize ([(config-parameter 'strict_proxy) "no"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") + (controls)))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") + `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") + (controls))))) + (check-equal? (get-media-html "https://example.com" "who knows") `"") + (check-equal? (get-media-html "https://example.com" #f) `"")) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-not-false + ((query-selector + (attribute-selector 'src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fexamplefile") + (generate-results-page #:source-url "" + #:wikiname "test" + #:title "File:Example file" + #:media-detail test-media-detail + #:image-content-type "image/jpeg")))))) From 58d6a652d872610818c1cf3be903052785e4778e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Oct 2022 21:59:02 +1300 Subject: [PATCH 045/166] Style tweaks to file page, add copyright notice --- src/page-file.rkt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 5c3f8965..e312e310 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -79,23 +79,24 @@ (p ,(if (non-empty-string? video-embed-code) `"" `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) - "Added by " + "Uploaded by " (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) - "." - ,(if is-posted-in - `(span " Posted in " - ,@(map (λ (article) - (define page-path (jp "/title" article)) - (define title (jp "/titleText" article page-path)) - `(span ,(if (eq? (car smaller-article-list) article) "" ", ") - (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title))) - smaller-article-list) - ,(if (eq? article-list-is-smaller 1) "…" ".")) - `"")) + ".") ,(if (string? image-description) (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) - "")))) + ; file license info might be displayed in the description, example: /lgbtqia/wiki/File:Rainbow_Flag1.svg + `(p "This file is likely protected by copyright. Consider the file's license and fair use law before reusing it.")) + ,(if is-posted-in + `(p "This file is used in " + ,@(map (λ (article) + (define page-path (jp "/title" article)) + (define title (jp "/titleText" article page-path)) + `(span ,(if (eq? (car smaller-article-list) article) "" ", ") + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title))) + smaller-article-list) + ,(if (eq? article-list-is-smaller 1) "…" ".")) + `"")))) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From 7a8a1cd40fed3b2a05730df37b6e81f8028bf05c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Oct 2022 22:52:35 +1300 Subject: [PATCH 046/166] Remove noscript versions of images Script versions are likely better quality, and BreezeWiki makes them viewable without requiring scripts anyway. --- src/page-wiki.rkt | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index effa40a0..07f2f817 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -51,6 +51,17 @@ (p "Another page link: " (a (@ (data-test-wikilink) (href "https://test.fandom.com/wiki/Another_Page") (title "Another Page")) "Another Page")))) + (figure (@ (class "thumb tnone")) + (a (@ (href "https://static.wikia.nocookie.net/nice-image.png") (class "image")) + (img (@ (src "data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D") + (data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") + (class "thumbimage lazyload")))) + (noscript + (a (@ (href "https://static.wikia.nocookie.net/nice-image.png") (class "image")) + (img (@ (src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") + (data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") + (class "thumbimage"))))) + (figcaption "Test figure!")) (iframe (@ (src "https://example.com/iframe-src"))))))) (define (preprocess-html-wiki html) @@ -108,6 +119,15 @@ `(a ((class "iframe-alternative") (href ,src)) (,(format "Embedded media: ~a" src)))] + ; remove noscript versions of images because they are likely lower quality than the script versions + [(and (eq? element-type 'noscript) + (match children + ; either the noscript has a.image as a first child... + [(list (list 'a (list '@ a-att ...) _)) (has-class? "image" a-att)] + ; or the noscript has img as a first child + [(list (list 'img _)) #t] + [_ #f])) + return-no-element] [#t (list element-type ;; attributes @@ -233,7 +253,9 @@ ((query-selector (λ (t a c) (and (eq? t 'a) (has-class? "image-thumbnail" a))) transformed)))) - "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image.png")) + "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image.png") + ; check that noscript images are removed + (check-equal? ((query-selector (λ (t a c) (eq? t 'noscript)) transformed)) #f)) (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From 4968dc2a49bf0d2aa0c5ac941fe6a1df4901bac4 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Oct 2022 22:54:05 +1300 Subject: [PATCH 047/166] strict proxy false in test as requested by blankie --- src/page-file.rkt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index e312e310..a8a41a88 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -146,12 +146,12 @@ (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") (controls))))) - (parameterize ([(config-parameter 'strict_proxy) "no"]) + (parameterize ([(config-parameter 'strict_proxy) "false"]) (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") - `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") + `(audio (@ (src "https://static.wikia.nocookie.net/c") (controls)))) (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") - `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") + `(video (@ (src "https://static.wikia.nocookie.net/d") (controls))))) (check-equal? (get-media-html "https://example.com" "who knows") `"") (check-equal? (get-media-html "https://example.com" #f) `"")) From 5b4efdd292d84028a4222c3592d86b547db96574 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Oct 2022 22:40:14 +1300 Subject: [PATCH 048/166] Add a funny redirect that I can make use of later --- breezewiki.rkt | 2 ++ dist.rkt | 2 ++ src/dispatcher-tree.rkt | 1 + src/page-home.rkt | 2 +- src/page-it-works.rkt | 15 +++++++++++++++ 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/page-it-works.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index 3fc9b8fb..a8b8c288 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -12,6 +12,7 @@ (require-reloadable "src/page-category.rkt" page-category) (require-reloadable "src/page-global-search.rkt" page-global-search) (require-reloadable "src/page-home.rkt" page-home) +(require-reloadable "src/page-it-works.rkt" page-it-works) (require-reloadable "src/page-not-found.rkt" page-not-found) (require-reloadable "src/page-proxy.rkt" page-proxy) (require-reloadable "src/page-redirect-wiki-home.rkt" redirect-wiki-home) @@ -35,6 +36,7 @@ page-category page-global-search page-home + page-it-works page-not-found page-proxy page-search diff --git a/dist.rkt b/dist.rkt index 805df48a..777e81ac 100644 --- a/dist.rkt +++ b/dist.rkt @@ -6,6 +6,7 @@ (require (only-in "src/page-category.rkt" page-category)) (require (only-in "src/page-global-search.rkt" page-global-search)) (require (only-in "src/page-home.rkt" page-home)) +(require (only-in "src/page-it-works.rkt" page-it-works)) (require (only-in "src/page-not-found.rkt" page-not-found)) (require (only-in "src/page-proxy.rkt" page-proxy)) (require (only-in "src/page-redirect-wiki-home.rkt" redirect-wiki-home)) @@ -24,6 +25,7 @@ page-category page-global-search page-home + page-it-works page-not-found page-proxy page-search diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index b68cf9c7..9e072bc8 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -44,6 +44,7 @@ (pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) + (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) diff --git a/src/page-home.rkt b/src/page-home.rkt index b16f66a1..86c316d1 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -49,7 +49,7 @@ ,(apply format "~a: ~a" x)))) examples)) (h2 "Testimonials") - (p (@ (class "testimonial")) ">So glad to never have to touch fandom's garbage platform directly ever again —RNL") + (p (@ (class "testimonial")) ">so glad someone introduced me to a F*ndom alternative (BreezeWiki) because that x-factorized spillway of an ad-infested radioactive dumpsite can go die in a fire —RB") (p (@ (class "testimonial")) ">you are so right that fandom still sucks even with adblock somehow. even zapping all the stupid padding it still sucks —Minimus") (p (@ (class "testimonial")) ">attempting to go to a wiki's forum page with breezewiki doesn't work, which is based honestly —Tom Skeleton") (p (@ (class "testimonial")) ">Fandom pages crashing and closing, taking forever to load and locking up as they load the ads on the site... they are causing the site to crash because they are trying to load video ads both at the top and bottom of the site as well as two or three banner ads, then a massive top of site ad and eventually my anti-virus shuts the whole site down because it's literally pulling more resources than WoW in ultra settings... —Anonymous") diff --git a/src/page-it-works.rkt b/src/page-it-works.rkt new file mode 100644 index 00000000..ce9e05f3 --- /dev/null +++ b/src/page-it-works.rkt @@ -0,0 +1,15 @@ +#lang racket/base +(require racket/dict + net/url + web-server/http + web-server/dispatchers/dispatch + "application-globals.rkt") + +(provide + page-it-works) + +(define (page-it-works req) + (define b? (dict-ref (url-query (request-uri req)) 'b #f)) + (if b? + (generate-redirect "/stampylongnose/wiki/It_Works") + (next-dispatcher))) From 2e6fa6088b91cfc1a920cc42e5866f141e8cbe4d Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 3 Oct 2022 21:56:17 +0700 Subject: [PATCH 049/166] Proxy links with class image Some images don't have an image-thumbnail class, for example: https://breezewiki.pussthecat.org/minecraft/wiki/Enchanting%20Table#Usage --- src/page-wiki.rkt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 07f2f817..6499a866 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -188,7 +188,8 @@ (λ (v) (and (config-true? 'strict_proxy) (eq? element-type 'a) - (has-class? "image-thumbnail" v))) + (or (has-class? "image-thumbnail" v) + (has-class? "image" v)))) (λ (v) (attribute-maybe-update 'href u-proxy-url v))) ; proxy images from src attributes, if strict_proxy is set (curry u From 9c05e95f0797d17afca3e2ca04a096cfd73c697e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Oct 2022 22:53:05 +1300 Subject: [PATCH 050/166] Test for proxy links with class image --- src/page-wiki.rkt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 6499a866..488886fa 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -52,7 +52,7 @@ (a (@ (data-test-wikilink) (href "https://test.fandom.com/wiki/Another_Page") (title "Another Page")) "Another Page")))) (figure (@ (class "thumb tnone")) - (a (@ (href "https://static.wikia.nocookie.net/nice-image.png") (class "image")) + (a (@ (href "https://static.wikia.nocookie.net/nice-image.png") (class "image") (data-test-figure-a)) (img (@ (src "data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D") (data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") (class "thumbimage lazyload")))) @@ -255,6 +255,11 @@ (λ (t a c) (and (eq? t 'a) (has-class? "image-thumbnail" a))) transformed)))) "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image.png") + (check-equal? (get-attribute 'href (bits->attributes + ((query-selector + (λ (t a c) (member '(data-test-figure-a) a)) + transformed)))) + "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnice-image.png") ; check that noscript images are removed (check-equal? ((query-selector (λ (t a c) (eq? t 'noscript)) transformed)) #f)) From 722b0589cb7449a4c5dfe9badc724954dbd03f2e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Oct 2022 22:54:46 +1300 Subject: [PATCH 051/166] Update minimum wikiname length to 1 Example: https://m.fandom.com/no/wiki/Forside --- src/url-utils.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/url-utils.rkt b/src/url-utils.rkt index 10df0891..b70b245f 100644 --- a/src/url-utils.rkt +++ b/src/url-utils.rkt @@ -22,7 +22,7 @@ (module+ test (require "typed-rackunit.rkt")) -(define px-wikiname "[a-zA-Z0-9-]{3,50}") +(define px-wikiname "[a-zA-Z0-9-]{1,50}") ;; https://url.spec.whatwg.org/#urlencoded-serializing From 71705d6e7486f7cfce4e58a821d3e87372ec9e34 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 10 Oct 2022 17:04:11 +0700 Subject: [PATCH 052/166] Fix used in links that have a namespace in file pages Example: /ben10/wiki/File:OS_Timespan.png https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Title.php#L1266 --- src/page-file.rkt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index a8a41a88..1802568e 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -31,8 +31,7 @@ (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") (userName . "blankie") (isPostedIn . #t) - (smallerArticleList . (#hasheq((title . "Example_article") - (titleText . "Example article")))) + (smallerArticleList . (#hasheq((titleText . "Test:Example article")))) (articleListIsSmaller . 0) (exists . #t) (imageDescription . #f)))) @@ -89,8 +88,8 @@ ,(if is-posted-in `(p "This file is used in " ,@(map (λ (article) - (define page-path (jp "/title" article)) - (define title (jp "/titleText" article page-path)) + (define title (jp "/titleText" article)) + (define page-path (regexp-replace* #rx" " title "_")) `(span ,(if (eq? (car smaller-article-list) article) "" ", ") (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) ,title))) From 07db44e732ea0c114f18014544efb47ddc48caf8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Oct 2022 00:26:06 +1300 Subject: [PATCH 053/166] Add search suggestions --- src/application-globals.rkt | 16 +++++-- src/page-static.rkt | 1 + static/main.css | 60 +++++++++++++++++++++++ static/preact.js | 21 ++++++++ static/search-suggestions.js | 93 ++++++++++++++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 static/preact.js create mode 100644 static/search-suggestions.js diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 7f16bee7..937a8548 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -1,5 +1,6 @@ #lang racket/base (require racket/string + json (prefix-in easy: net/http-easy) html-writing web-server/http @@ -94,7 +95,11 @@ ,@(map (λ (url) `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) (required-styles (format "https://~a.fandom.com" wikiname))) - (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css")))) + (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css"))) + (script "const BWData = " + ,(jsexpr->string (hasheq 'wikiname wikiname + 'strict_proxy (config-true? 'strict_proxy)))) + (script (@ (type "module") (src "/static/search-suggestions.js")))) (body (@ (class ,body-class)) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileHorizontally header"))) @@ -103,9 +108,12 @@ (div (@ (class "custom-top")) (h1 (@ (class "page-title")) ,title) (nav (@ (class "sitesearch")) - (form (@ (action ,(format "/~a/search" wikiname))) - (label "Search " - (input (@ (type "text") (name "q"))))))) + (form (@ (action ,(format "/~a/search" wikiname)) + (class "bw-search-form") + (id "bw-pr-search")) + (label (@ (for "bw-search-input")) "Search ") + (input (@ (type "text") (name "q") (id "bw-search-input") (autocomplete "off"))) + (div (@ (class "bw-ss__container")))))) (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,content)) diff --git a/src/page-static.rkt b/src/page-static.rkt index e684c745..353a0f86 100644 --- a/src/page-static.rkt +++ b/src/page-static.rkt @@ -19,6 +19,7 @@ (define hash-ext-mime-type (hash #".css" #"text/css" + #".js" #"text/javascript" #".png" #"image/png" #".svg" #"image/svg+xml" #".txt" #"text/plain")) diff --git a/static/main.css b/static/main.css index ba0eaadf..c773db78 100644 --- a/static/main.css +++ b/static/main.css @@ -225,6 +225,66 @@ figcaption, .lightbox-caption, .thumbcaption { margin-left: 1.2em; } +/* (breezewiki) search suggestions */ +.bw-search-form { + display: grid; + grid-template-columns: auto 1fr; + grid-gap: 0px 5px; +} +.bw-ss__container { + grid-column: 2; + position: relative; +} +.bw-ss__list { + position: absolute; + left: 0; + right: 0; + list-style-type: none; + padding: 0; + margin: 0; + font-size: 14px; + background: white; + color: black; + border: solid #808080; + border-width: 0px 1px 1px; + box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.5); + z-index: 99; + + display: none; +} +.bw-ss__list--focus { + display: block; +} +.bw-ss__list--loading { + background: #c0c0c0; +} +.bw-ss__input--accepted { + background: #fffbc0; +} +.bw-ss__preview { + padding: 0px 2px; + font-style: italic; + color: #555; +} +.bw-ss__item { + display: grid; /* make buttons take the full size */ +} +.bw-ss__item:hover { + background-color: #ddd; +} +.bw-ss__button { + appearance: none; + -moz-appearance: none; + border: none; + margin: 0; + line-height: inherit; + background: none; + font: inherit; + cursor: pointer; + padding: 0px 2px; + text-align: left; +} + /* media queries */ /* for reference, cell phone screens are generally 400 px wide, definitely less than 500 px */ diff --git a/static/preact.js b/static/preact.js new file mode 100644 index 00000000..0ba46a47 --- /dev/null +++ b/static/preact.js @@ -0,0 +1,21 @@ +// preact +const {Component, Fragment, cloneElement, createContext, createElement, createRef, h, hydrate, isValidElement, options, render, toChildArray} = (function() { + var n,l,u,i,t,o,r,f = {},e = [],c = /acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i;function s(n, l) {for (var u in l) n[u] = l[u];return n;}function a(n) {var l = n.parentNode;l && l.removeChild(n);}function h(l, u, i) {var t,o,r,f = {};for (r in u) "key" == r ? t = u[r] : "ref" == r ? o = u[r] : f[r] = u[r];if (arguments.length > 2 && (f.children = arguments.length > 3 ? n.call(arguments, 2) : i), "function" == typeof l && null != l.defaultProps) for (r in l.defaultProps) void 0 === f[r] && (f[r] = l.defaultProps[r]);return v(l, f, t, o, null);}function v(n, i, t, o, r) {var f = { type: n, props: i, key: t, ref: o, __k: null, __: null, __b: 0, __e: null, __d: void 0, __c: null, __h: null, constructor: void 0, __v: null == r ? ++u : r };return null == r && null != l.vnode && l.vnode(f), f;}function y() {return { current: null };}function p(n) {return n.children;}function d(n, l) {this.props = n, this.context = l;}function _(n, l) {if (null == l) return n.__ ? _(n.__, n.__.__k.indexOf(n) + 1) : null;for (var u; l < n.__k.length; l++) if (null != (u = n.__k[l]) && null != u.__e) return u.__e;return "function" == typeof n.type ? _(n) : null;}function k(n) {var l, u;if (null != (n = n.__) && null != n.__c) {for (n.__e = n.__c.base = null, l = 0; l < n.__k.length; l++) if (null != (u = n.__k[l]) && null != u.__e) {n.__e = n.__c.base = u.__e;break;}return k(n);}}function b(n) {(!n.__d && (n.__d = !0) && t.push(n) && !g.__r++ || o !== l.debounceRendering) && ((o = l.debounceRendering) || setTimeout)(g);}function g() {for (var n; g.__r = t.length;) n = t.sort(function (n, l) {return n.__v.__b - l.__v.__b;}), t = [], n.some(function (n) {var l, u, i, t, o, r;n.__d && (o = (t = (l = n).__v).__e, (r = l.__P) && (u = [], (i = s({}, t)).__v = t.__v + 1, j(r, t, i, l.__n, void 0 !== r.ownerSVGElement, null != t.__h ? [o] : null, u, null == o ? _(t) : o, t.__h), z(u, t), t.__e != o && k(t)));});}function w(n, l, u, i, t, o, r, c, s, a) {var h,y,d,k,b,g,w,x = i && i.__k || e,C = x.length;for (u.__k = [], h = 0; h < l.length; h++) if (null != (k = u.__k[h] = null == (k = l[h]) || "boolean" == typeof k ? null : "string" == typeof k || "number" == typeof k || "bigint" == typeof k ? v(null, k, null, null, k) : Array.isArray(k) ? v(p, { children: k }, null, null, null) : k.__b > 0 ? v(k.type, k.props, k.key, k.ref ? k.ref : null, k.__v) : k)) {if (k.__ = u, k.__b = u.__b + 1, null === (d = x[h]) || d && k.key == d.key && k.type === d.type) x[h] = void 0;else for (y = 0; y < C; y++) {if ((d = x[y]) && k.key == d.key && k.type === d.type) {x[y] = void 0;break;}d = null;}j(n, k, d = d || f, t, o, r, c, s, a), b = k.__e, (y = k.ref) && d.ref != y && (w || (w = []), d.ref && w.push(d.ref, null, k), w.push(y, k.__c || b, k)), null != b ? (null == g && (g = b), "function" == typeof k.type && k.__k === d.__k ? k.__d = s = m(k, s, n) : s = A(n, k, d, x, b, s), "function" == typeof u.type && (u.__d = s)) : s && d.__e == s && s.parentNode != n && (s = _(d));}for (u.__e = g, h = C; h--;) null != x[h] && N(x[h], x[h]);if (w) for (h = 0; h < w.length; h++) M(w[h], w[++h], w[++h]);}function m(n, l, u) {for (var i, t = n.__k, o = 0; t && o < t.length; o++) (i = t[o]) && (i.__ = n, l = "function" == typeof i.type ? m(i, l, u) : A(u, i, i, t, i.__e, l));return l;}function x(n, l) {return l = l || [], null == n || "boolean" == typeof n || (Array.isArray(n) ? n.some(function (n) {x(n, l);}) : l.push(n)), l;}function A(n, l, u, i, t, o) {var r, f, e;if (void 0 !== l.__d) r = l.__d, l.__d = void 0;else if (null == u || t != o || null == t.parentNode) n: if (null == o || o.parentNode !== n) n.appendChild(t), r = null;else {for (f = o, e = 0; (f = f.nextSibling) && e < i.length; e += 2) if (f == t) break n;n.insertBefore(t, o), r = o;}return void 0 !== r ? r : t.nextSibling;}function C(n, l, u, i, t) {var o;for (o in u) "children" === o || "key" === o || o in l || H(n, o, null, u[o], i);for (o in l) t && "function" != typeof l[o] || "children" === o || "key" === o || "value" === o || "checked" === o || u[o] === l[o] || H(n, o, l[o], u[o], i);}function $(n, l, u) {"-" === l[0] ? n.setProperty(l, u) : n[l] = null == u ? "" : "number" != typeof u || c.test(l) ? u : u + "px";}function H(n, l, u, i, t) {var o;n: if ("style" === l) {if ("string" == typeof u) n.style.cssText = u;else {if ("string" == typeof i && (n.style.cssText = i = ""), i) for (l in i) u && l in u || $(n.style, l, "");if (u) for (l in u) i && u[l] === i[l] || $(n.style, l, u[l]);}} else if ("o" === l[0] && "n" === l[1]) o = l !== (l = l.replace(/Capture$/, "")), l = l.toLowerCase() in n ? l.toLowerCase().slice(2) : l.slice(2), n.l || (n.l = {}), n.l[l + o] = u, u ? i || n.addEventListener(l, o ? T : I, o) : n.removeEventListener(l, o ? T : I, o);else if ("dangerouslySetInnerHTML" !== l) {if (t) l = l.replace(/xlink(H|:h)/, "h").replace(/sName$/, "s");else if ("href" !== l && "list" !== l && "form" !== l && "tabIndex" !== l && "download" !== l && l in n) try {n[l] = null == u ? "" : u;break n;} catch (n) {}"function" == typeof u || (null == u || !1 === u && -1 == l.indexOf("-") ? n.removeAttribute(l) : n.setAttribute(l, u));}}function I(n) {this.l[n.type + !1](l.event ? l.event(n) : n);}function T(n) {this.l[n.type + !0](l.event ? l.event(n) : n);}function j(n, u, i, t, o, r, f, e, c) {var a,h,v,y,_,k,b,g,m,x,A,C,$,H,I,T = u.type;if (void 0 !== u.constructor) return null;null != i.__h && (c = i.__h, e = u.__e = i.__e, u.__h = null, r = [e]), (a = l.__b) && a(u);try {n: if ("function" == typeof T) {if (g = u.props, m = (a = T.contextType) && t[a.__c], x = a ? m ? m.props.value : a.__ : t, i.__c ? b = (h = u.__c = i.__c).__ = h.__E : ("prototype" in T && T.prototype.render ? u.__c = h = new T(g, x) : (u.__c = h = new d(g, x), h.constructor = T, h.render = O), m && m.sub(h), h.props = g, h.state || (h.state = {}), h.context = x, h.__n = t, v = h.__d = !0, h.__h = [], h._sb = []), null == h.__s && (h.__s = h.state), null != T.getDerivedStateFromProps && (h.__s == h.state && (h.__s = s({}, h.__s)), s(h.__s, T.getDerivedStateFromProps(g, h.__s))), y = h.props, _ = h.state, v) null == T.getDerivedStateFromProps && null != h.componentWillMount && h.componentWillMount(), null != h.componentDidMount && h.__h.push(h.componentDidMount);else {if (null == T.getDerivedStateFromProps && g !== y && null != h.componentWillReceiveProps && h.componentWillReceiveProps(g, x), !h.__e && null != h.shouldComponentUpdate && !1 === h.shouldComponentUpdate(g, h.__s, x) || u.__v === i.__v) {for (h.props = g, h.state = h.__s, u.__v !== i.__v && (h.__d = !1), h.__v = u, u.__e = i.__e, u.__k = i.__k, u.__k.forEach(function (n) {n && (n.__ = u);}), A = 0; A < h._sb.length; A++) h.__h.push(h._sb[A]);h._sb = [], h.__h.length && f.push(h);break n;}null != h.componentWillUpdate && h.componentWillUpdate(g, h.__s, x), null != h.componentDidUpdate && h.__h.push(function () {h.componentDidUpdate(y, _, k);});}if (h.context = x, h.props = g, h.__v = u, h.__P = n, C = l.__r, $ = 0, "prototype" in T && T.prototype.render) {for (h.state = h.__s, h.__d = !1, C && C(u), a = h.render(h.props, h.state, h.context), H = 0; H < h._sb.length; H++) h.__h.push(h._sb[H]);h._sb = [];} else do {h.__d = !1, C && C(u), a = h.render(h.props, h.state, h.context), h.state = h.__s;} while (h.__d && ++$ < 25);h.state = h.__s, null != h.getChildContext && (t = s(s({}, t), h.getChildContext())), v || null == h.getSnapshotBeforeUpdate || (k = h.getSnapshotBeforeUpdate(y, _)), I = null != a && a.type === p && null == a.key ? a.props.children : a, w(n, Array.isArray(I) ? I : [I], u, i, t, o, r, f, e, c), h.base = u.__e, u.__h = null, h.__h.length && f.push(h), b && (h.__E = h.__ = null), h.__e = !1;} else null == r && u.__v === i.__v ? (u.__k = i.__k, u.__e = i.__e) : u.__e = L(i.__e, u, i, t, o, r, f, c);(a = l.diffed) && a(u);} catch (n) {u.__v = null, (c || null != r) && (u.__e = e, u.__h = !!c, r[r.indexOf(e)] = null), l.__e(n, u, i);}}function z(n, u) {l.__c && l.__c(u, n), n.some(function (u) {try {n = u.__h, u.__h = [], n.some(function (n) {n.call(u);});} catch (n) {l.__e(n, u.__v);}});}function L(l, u, i, t, o, r, e, c) {var s,h,v,y = i.props,p = u.props,d = u.type,k = 0;if ("svg" === d && (o = !0), null != r) for (; k < r.length; k++) if ((s = r[k]) && "setAttribute" in s == !!d && (d ? s.localName === d : 3 === s.nodeType)) {l = s, r[k] = null;break;}if (null == l) {if (null === d) return document.createTextNode(p);l = o ? document.createElementNS("http://www.w3.org/2000/svg", d) : document.createElement(d, p.is && p), r = null, c = !1;}if (null === d) y === p || c && l.data === p || (l.data = p);else {if (r = r && n.call(l.childNodes), h = (y = i.props || f).dangerouslySetInnerHTML, v = p.dangerouslySetInnerHTML, !c) {if (null != r) for (y = {}, k = 0; k < l.attributes.length; k++) y[l.attributes[k].name] = l.attributes[k].value;(v || h) && (v && (h && v.__html == h.__html || v.__html === l.innerHTML) || (l.innerHTML = v && v.__html || ""));}if (C(l, p, y, o, c), v) u.__k = [];else if (k = u.props.children, w(l, Array.isArray(k) ? k : [k], u, i, t, o && "foreignObject" !== d, r, e, r ? r[0] : i.__k && _(i, 0), c), null != r) for (k = r.length; k--;) null != r[k] && a(r[k]);c || ("value" in p && void 0 !== (k = p.value) && (k !== l.value || "progress" === d && !k || "option" === d && k !== y.value) && H(l, "value", k, y.value, !1), "checked" in p && void 0 !== (k = p.checked) && k !== l.checked && H(l, "checked", k, y.checked, !1));}return l;}function M(n, u, i) {try {"function" == typeof n ? n(u) : n.current = u;} catch (n) {l.__e(n, i);}}function N(n, u, i) {var t, o;if (l.unmount && l.unmount(n), (t = n.ref) && (t.current && t.current !== n.__e || M(t, null, u)), null != (t = n.__c)) {if (t.componentWillUnmount) try {t.componentWillUnmount();} catch (n) {l.__e(n, u);}t.base = t.__P = null, n.__c = void 0;}if (t = n.__k) for (o = 0; o < t.length; o++) t[o] && N(t[o], u, i || "function" != typeof n.type);i || null == n.__e || a(n.__e), n.__ = n.__e = n.__d = void 0;}function O(n, l, u) {return this.constructor(n, u);}function P(u, i, t) {var o, r, e;l.__ && l.__(u, i), r = (o = "function" == typeof t) ? null : t && t.__k || i.__k, e = [], j(i, u = (!o && t || i).__k = h(p, null, [u]), r || f, f, void 0 !== i.ownerSVGElement, !o && t ? [t] : r ? null : i.firstChild ? n.call(i.childNodes) : null, e, !o && t ? t : r ? r.__e : i.firstChild, o), z(e, u);}function S(n, l) {P(n, l, S);}function q(l, u, i) {var t,o,r,f = s({}, l.props);for (r in u) "key" == r ? t = u[r] : "ref" == r ? o = u[r] : f[r] = u[r];return arguments.length > 2 && (f.children = arguments.length > 3 ? n.call(arguments, 2) : i), v(l.type, f, t || l.key, o || l.ref, null);}function B(n, l) {var u = { __c: l = "__cC" + r++, __: n, Consumer: function (n, l) {return n.children(l);}, Provider: function (n) {var u, i;return this.getChildContext || (u = [], (i = {})[l] = this, this.getChildContext = function () {return i;}, this.shouldComponentUpdate = function (n) {this.props.value !== n.value && u.some(b);}, this.sub = function (n) {u.push(n);var l = n.componentWillUnmount;n.componentWillUnmount = function () {u.splice(u.indexOf(n), 1), l && l.call(n);};}), n.children;} };return u.Provider.__ = u.Consumer.contextType = u;}n = e.slice, l = { __e: function (n, l, u, i) {for (var t, o, r; l = l.__;) if ((t = l.__c) && !t.__) try {if ((o = t.constructor) && null != o.getDerivedStateFromError && (t.setState(o.getDerivedStateFromError(n)), r = t.__d), null != t.componentDidCatch && (t.componentDidCatch(n, i || {}), r = t.__d), r) return t.__E = t;} catch (l) {n = l;}throw n;} }, u = 0, i = function (n) {return null != n && void 0 === n.constructor;}, d.prototype.setState = function (n, l) {var u;u = null != this.__s && this.__s !== this.state ? this.__s : this.__s = s({}, this.state), "function" == typeof n && (n = n(s({}, u), this.props)), n && s(u, n), null != n && this.__v && (l && this._sb.push(l), b(this));}, d.prototype.forceUpdate = function (n) {this.__v && (this.__e = !0, n && this.__h.push(n), b(this));}, d.prototype.render = p, t = [], g.__r = 0, r = 0; + return { Component: d, Fragment: p, cloneElement: q, createContext: B, createElement: h, createRef: y, h, hydrate: S, isValidElement: i, options: l, render: P, toChildArray: x }; +})(); +export { Component, Fragment, cloneElement, createContext, createElement, createRef, h, hydrate, isValidElement, options, render, toChildArray }; + +// htm +const htm = (function() { + var n = function (t, s, r, e) {var u;s[0] = 0;for (var h = 1; h < s.length; h++) {var p = s[h++],a = s[h] ? (s[0] |= p ? 1 : 2, r[s[h++]]) : s[++h];3 === p ? e[0] = a : 4 === p ? e[1] = Object.assign(e[1] || {}, a) : 5 === p ? (e[1] = e[1] || {})[s[++h]] = a : 6 === p ? e[1][s[++h]] += a + "" : p ? (u = t.apply(a, n(t, a, r, ["", null])), e.push(u), a[0] ? s[0] |= 2 : (s[h - 2] = 0, s[h] = u)) : e.push(a);}return e;},t = new Map(); + return function (s) {var r = t.get(this);return r || (r = new Map(), t.set(this, r)), (r = n(this, r.get(s) || (r.set(s, r = function (n) {for (var t, s, r = 1, e = "", u = "", h = [0], p = function (n) {1 === r && (n || (e = e.replace(/^\s*\n\s*|\s*\n\s*$/g, ""))) ? h.push(0, n, e) : 3 === r && (n || e) ? (h.push(3, n, e), r = 2) : 2 === r && "..." === e && n ? h.push(4, n, 0) : 2 === r && e && !n ? h.push(5, 0, !0, e) : r >= 5 && ((e || !n && 5 === r) && (h.push(r, 0, e, s), r = 6), n && (h.push(r, n, 0, s), r = 6)), e = "";}, a = 0; a < n.length; a++) {a && (1 === r && p(), p(a));for (var l = 0; l < n[a].length; l++) t = n[a][l], 1 === r ? "<" === t ? (p(), h = [h], r = 3) : e += t : 4 === r ? "--" === e && ">" === t ? (r = 1, e = "") : e = t + e[0] : u ? t === u ? u = "" : e += t : '"' === t || "'" === t ? u = t : ">" === t ? (p(), r = 1) : r && ("=" === t ? (r = 5, s = e, e = "") : "/" === t && (r < 5 || ">" === n[a][l + 1]) ? (p(), 3 === r && (h = h[0]), r = h, (h = h[0]).push(2, 0, r), r = 0) : " " === t || "\t" === t || "\n" === t || "\r" === t ? (p(), r = 2) : e += t), 3 === r && "!--" === e && (r = 4, h = h[0]);}return p(), h;}(s)), r), arguments, [])).length > 1 ? r : r[0];} +})(); +export {htm}; + +// hooks +const { useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useState } = (function() { + var n = options; + var t,r,u,i,o = 0,f = [],c = [],e = n.__b,a = n.__r,v = n.diffed,l = n.__c,m = n.unmount;function d(t, u) {n.__h && n.__h(r, t, o || u), o = 0;var i = r.__H || (r.__H = { __: [], __h: [] });return t >= i.__.length && i.__.push({ __V: c }), i.__[t];}function p(n) {return o = 1, y(B, n);}function y(n, u, i) {var o = d(t++, 2);if (o.t = n, !o.__c && (o.__ = [i ? i(u) : B(void 0, u), function (n) {var t = o.__N ? o.__N[0] : o.__[0],r = o.t(t, n);t !== r && (o.__N = [r, o.__[1]], o.__c.setState({}));}], o.__c = r, !r.u)) {r.u = !0;var f = r.shouldComponentUpdate;r.shouldComponentUpdate = function (n, t, r) {if (!o.__c.__H) return !0;var u = o.__c.__H.__.filter(function (n) {return n.__c;});if (u.every(function (n) {return !n.__N;})) return !f || f.call(this, n, t, r);var i = !1;return u.forEach(function (n) {if (n.__N) {var t = n.__[0];n.__ = n.__N, n.__N = void 0, t !== n.__[0] && (i = !0);}}), !(!i && o.__c.props === n) && (!f || f.call(this, n, t, r));};}return o.__N || o.__;}function h(u, i) {var o = d(t++, 3);!n.__s && z(o.__H, i) && (o.__ = u, o.i = i, r.__H.__h.push(o));}function s(u, i) {var o = d(t++, 4);!n.__s && z(o.__H, i) && (o.__ = u, o.i = i, r.__h.push(o));}function _(n) {return o = 5, F(function () {return { current: n };}, []);}function A(n, t, r) {o = 6, s(function () {return "function" == typeof n ? (n(t()), function () {return n(null);}) : n ? (n.current = t(), function () {return n.current = null;}) : void 0;}, null == r ? r : r.concat(n));}function F(n, r) {var u = d(t++, 7);return z(u.__H, r) ? (u.__V = n(), u.i = r, u.__h = n, u.__V) : u.__;}function T(n, t) {return o = 8, F(function () {return n;}, t);}function q(n) {var u = r.context[n.__c],i = d(t++, 9);return i.c = n, u ? (null == i.__ && (i.__ = !0, u.sub(r)), u.props.value) : n.__;}function x(t, r) {n.useDebugValue && n.useDebugValue(r ? r(t) : t);}function P(n) {var u = d(t++, 10),i = p();return u.__ = n, r.componentDidCatch || (r.componentDidCatch = function (n, t) {u.__ && u.__(n, t), i[1](n);}), [i[0], function () {i[1](void 0);}];}function V() {var n = d(t++, 11);return n.__ || (n.__ = "P" + function (n) {for (var t = 0, r = n.length; r > 0;) t = (t << 5) - t + n.charCodeAt(--r) | 0;return t;}(r.__v.__m) + t), n.__;}function b() {for (var t; t = f.shift();) if (t.__P && t.__H) try {t.__H.__h.forEach(k), t.__H.__h.forEach(w), t.__H.__h = [];} catch (r) {t.__H.__h = [], n.__e(r, t.__v);}}n.__b = function (n) {"function" != typeof n.type || n.__m || null === n.__ ? n.__m || (n.__m = n.__ && n.__.__m ? n.__.__m : "") : n.__m = (n.__ && n.__.__m ? n.__.__m : "") + (n.__ && n.__.__k ? n.__.__k.indexOf(n) : 0), r = null, e && e(n);}, n.__r = function (n) {a && a(n), t = 0;var i = (r = n.__c).__H;i && (u === r ? (i.__h = [], r.__h = [], i.__.forEach(function (n) {n.__N && (n.__ = n.__N), n.__V = c, n.__N = n.i = void 0;})) : (i.__h.forEach(k), i.__h.forEach(w), i.__h = [])), u = r;}, n.diffed = function (t) {v && v(t);var o = t.__c;o && o.__H && (o.__H.__h.length && (1 !== f.push(o) && i === n.requestAnimationFrame || ((i = n.requestAnimationFrame) || j)(b)), o.__H.__.forEach(function (n) {n.i && (n.__H = n.i), n.__V !== c && (n.__ = n.__V), n.i = void 0, n.__V = c;})), u = r = null;}, n.__c = function (t, r) {r.some(function (t) {try {t.__h.forEach(k), t.__h = t.__h.filter(function (n) {return !n.__ || w(n);});} catch (u) {r.some(function (n) {n.__h && (n.__h = []);}), r = [], n.__e(u, t.__v);}}), l && l(t, r);}, n.unmount = function (t) {m && m(t);var r,u = t.__c;u && u.__H && (u.__H.__.forEach(function (n) {try {k(n);} catch (n) {r = n;}}), u.__H = void 0, r && n.__e(r, u.__v));};var g = "function" == typeof requestAnimationFrame;function j(n) {var t,r = function () {clearTimeout(u), g && cancelAnimationFrame(t), setTimeout(n);},u = setTimeout(r, 100);g && (t = requestAnimationFrame(r));}function k(n) {var t = r,u = n.__c;"function" == typeof u && (n.__c = void 0, u()), r = t;}function w(n) {var t = r;n.__c = n.__(), r = t;}function z(n, t) {return !n || n.length !== t.length || t.some(function (t, r) {return t !== n[r];});}function B(n, t) {return "function" == typeof t ? t(n) : t;} + return { useCallback: T, useContext: q, useDebugValue: x, useEffect: h, useErrorBoundary: P, useId: V, useImperativeHandle: A, useLayoutEffect: s, useMemo: F, useReducer: y, useRef: _, useState: p }; +})(); +export { useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useState }; diff --git a/static/search-suggestions.js b/static/search-suggestions.js new file mode 100644 index 00000000..b3d08040 --- /dev/null +++ b/static/search-suggestions.js @@ -0,0 +1,93 @@ +import {h, htm, render, useState, useEffect, createContext, useContext} from "./preact.js" +const html = htm.bind(h) +const classNames = classArr => classArr.filter(el => el).join(" ") + +const form = document.getElementById("bw-pr-search") + +const AcceptSuggestion = createContext(null) +const hitsPromise = new Map() +const hitsDone = new Set() + +function Suggestion(props) { + const acceptSuggestion = useContext(AcceptSuggestion) + return html`
  • ` +} + +function fetchSuggestions(query, setSuggestions) { + if (query === "") query = "\0" + if (hitsPromise.has(query)) return hitsPromise.get(query) + const url = new URL(`https://${BWData.wikiname}.fandom.com/api.php`) + url.searchParams.set("action", "opensearch") + url.searchParams.set("format", "json") + url.searchParams.set("namespace", "0") // wiki namespace, 0 is default + url.searchParams.set("origin", "*") // mediawiki api cors + url.searchParams.set("search", query) + const sendUrl = BWData.strict_proxy + ? "/proxy?" + new URLSearchParams({dest: url}) + : url + const promise = fetch(sendUrl).then(res => res.json()).then(root => { + hitsDone.add(query) + return Array(root[1].length).fill().map((_, i) => ({ + title: root[1][i], + url: root[3][i] + })) + }) + hitsPromise.set(query, promise) + return promise +} + +function SuggestionList(props) { + return html` +
    +
      + ${props.hits.map(hit => html`<${Suggestion} ...${hit} />`)} +
    +
    ` +} + +function ControlledInput() { + const [query, setQuery] = useState("") + const [focus, setFocus] = useState(false) + const [st, setSt] = useState("ready") + const [suggestions, setSuggestions] = useState([]) + + useEffect(() => { + if (st === "accepted") return + setSt("loading") + fetchSuggestions(query).then(s => { + setSuggestions(s) + if (hitsDone.size === hitsPromise.size) { + setSt("ready") + } + }) + }, [query]) + + function acceptSuggestion(suggestion) { + setQuery(suggestion.title) + setSt("accepted") + const dest = new URL(suggestion.url).pathname.match("/wiki/.*") + location = `/${BWData.wikiname}${dest}` + } + + useEffect(() => { + function listener(event) { + if (event.type === "focusin") setFocus(true) + else setFocus(false) + } + form.addEventListener("focusin", listener) + form.addEventListener("focusout", listener) + return () => { + form.removeEventListener("focusin", listener) + form.removeEventListener("focusout", listener) + } + }) + + return html` +<${AcceptSuggestion.Provider} value=${acceptSuggestion}> + + setQuery(e.target.value)} value=${query} class=${classNames(["bw-ss__input", `bw-ss__input--${st}`])} /> + <${SuggestionList} hits=${suggestions} focus=${focus} st=${st}/> +`, form) From e709b3cea55dbb485163a8034a943e92f2ef3826 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Oct 2022 00:35:34 +1300 Subject: [PATCH 054/166] New configuration: feature_search_suggestions Default true. Set this to false to disable search suggestions for the whole instance. --- src/application-globals.rkt | 4 +++- src/config.rkt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 937a8548..f2d83e9b 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -99,7 +99,9 @@ (script "const BWData = " ,(jsexpr->string (hasheq 'wikiname wikiname 'strict_proxy (config-true? 'strict_proxy)))) - (script (@ (type "module") (src "/static/search-suggestions.js")))) + ,(if (config-true? 'feature_search_suggestions) + '(script (@ (type "module") (src "/static/search-suggestions.js"))) + "")) (body (@ (class ,body-class)) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileHorizontally header"))) diff --git a/src/config.rkt b/src/config.rkt index 6fc029ff..bc0cf6c7 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -34,6 +34,7 @@ '((application_name . "BreezeWiki") (canonical_origin . "") (debug . "false") + (feature_search_suggestions . "true") (instance_is_official . "false") ; please don't turn this on, or you will make me very upset (log_outgoing . "true") (port . "10416") From bf055836ccb0f7d716ecbe031baea829ebad3f78 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 24 Oct 2022 00:22:47 +1300 Subject: [PATCH 055/166] Add download wiki names script --- misc/download-wiki-names.rkt | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 misc/download-wiki-names.rkt diff --git a/misc/download-wiki-names.rkt b/misc/download-wiki-names.rkt new file mode 100644 index 00000000..ff8fff97 --- /dev/null +++ b/misc/download-wiki-names.rkt @@ -0,0 +1,45 @@ +#lang racket/base +(require racket/generator + racket/list + racket/string + json + net/http-easy + html-parsing + "src/xexpr-utils.rkt" + "src/url-utils.rkt") + +(define output-file "wiki-names.json") +(define limit "5000") + +(define (get-page offset) + (define res (get (format "https://community.fandom.com/wiki/Special:NewWikis?~a" + (params->query `(("offset" . ,offset) + ("limit" . ,limit)))))) + (html->xexp (bytes->string/utf-8 (response-body res)))) + +(define (convert-list-items gen) + (for/list ([item (in-producer gen #f)]) + ; '(li "\n" "\t" (a (@ (href "http://terra-hexalis.fandom.com/")) "Terra Hexalis Wiki") "\n" "\t\t\ten\t") + (hasheq 'title (third (fourth item)) + 'link (second (second (second (fourth item)))) + 'lang (string-trim (sixth item))))) + +(define (get-items-recursive [offset ""] [items null]) + (define page (get-page offset)) + (define page-content ((query-selector (attribute-selector 'class "mw-spcontent") page))) + (define next ((query-selector (attribute-selector 'class "mw-nextlink") page-content))) + (define next-offset + (if next + (second (regexp-match #rx"offset=([0-9]*)" (get-attribute 'href (bits->attributes next)))) + #f)) + (define list-item-generator (query-selector (λ (e a c) (eq? e 'li)) page-content)) + (define these-items (convert-list-items list-item-generator)) + (define all-items (append items these-items)) + (printf "page offset \"~a\" has ~a items (~a so far)~n" offset (length these-items) (length all-items)) + (if next + (get-items-recursive next-offset all-items) + all-items)) + +(call-with-output-file output-file #:exists 'truncate/replace + (λ (out) + (write-json (get-items-recursive) out))) From d8d4e4375ed8f57639887dedd8e80cfd51cc71a7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 24 Oct 2022 18:13:14 +1300 Subject: [PATCH 056/166] Fix imports in wiki name downloader --- misc/download-wiki-names.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/download-wiki-names.rkt b/misc/download-wiki-names.rkt index ff8fff97..96a8f9a1 100644 --- a/misc/download-wiki-names.rkt +++ b/misc/download-wiki-names.rkt @@ -5,8 +5,8 @@ json net/http-easy html-parsing - "src/xexpr-utils.rkt" - "src/url-utils.rkt") + "../src/xexpr-utils.rkt" + "../src/url-utils.rkt") (define output-file "wiki-names.json") (define limit "5000") From 3c7a2f84530e9015e1ca18f857b998ce79cc2e48 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 24 Oct 2022 18:13:54 +1300 Subject: [PATCH 057/166] Rewrite search suggestions module --- src/application-globals.rkt | 7 +-- static/main.css | 2 +- static/preact.js | 19 ++++++- static/search-suggestions.js | 100 +++++++++++++++++------------------ 4 files changed, 71 insertions(+), 57 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index f2d83e9b..91c7e5a0 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -112,10 +112,11 @@ (nav (@ (class "sitesearch")) (form (@ (action ,(format "/~a/search" wikiname)) (class "bw-search-form") - (id "bw-pr-search")) + (id "bw-pr-search-form")) (label (@ (for "bw-search-input")) "Search ") - (input (@ (type "text") (name "q") (id "bw-search-input") (autocomplete "off"))) - (div (@ (class "bw-ss__container")))))) + (div (@ (id "bw-pr-search-input")) + (input (@ (type "text") (name "q") (id "bw-search-input") (autocomplete "off")))) + (div (@ (class "bw-ss__container") (id "bw-pr-search-suggestions")))))) (div (@ (id "content") #;(class "page-content")) (div (@ (id "mw-content-text")) ,content)) diff --git a/static/main.css b/static/main.css index c773db78..c4b6c209 100644 --- a/static/main.css +++ b/static/main.css @@ -269,7 +269,7 @@ figcaption, .lightbox-caption, .thumbcaption { .bw-ss__item { display: grid; /* make buttons take the full size */ } -.bw-ss__item:hover { +.bw-ss__button:hover, .bw-ss__button:focus { background-color: #ddd; } .bw-ss__button { diff --git a/static/preact.js b/static/preact.js index 0ba46a47..77d872c2 100644 --- a/static/preact.js +++ b/static/preact.js @@ -13,9 +13,24 @@ const htm = (function() { export {htm}; // hooks -const { useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useState } = (function() { +const { useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState } = (function() { var n = options; var t,r,u,i,o = 0,f = [],c = [],e = n.__b,a = n.__r,v = n.diffed,l = n.__c,m = n.unmount;function d(t, u) {n.__h && n.__h(r, t, o || u), o = 0;var i = r.__H || (r.__H = { __: [], __h: [] });return t >= i.__.length && i.__.push({ __V: c }), i.__[t];}function p(n) {return o = 1, y(B, n);}function y(n, u, i) {var o = d(t++, 2);if (o.t = n, !o.__c && (o.__ = [i ? i(u) : B(void 0, u), function (n) {var t = o.__N ? o.__N[0] : o.__[0],r = o.t(t, n);t !== r && (o.__N = [r, o.__[1]], o.__c.setState({}));}], o.__c = r, !r.u)) {r.u = !0;var f = r.shouldComponentUpdate;r.shouldComponentUpdate = function (n, t, r) {if (!o.__c.__H) return !0;var u = o.__c.__H.__.filter(function (n) {return n.__c;});if (u.every(function (n) {return !n.__N;})) return !f || f.call(this, n, t, r);var i = !1;return u.forEach(function (n) {if (n.__N) {var t = n.__[0];n.__ = n.__N, n.__N = void 0, t !== n.__[0] && (i = !0);}}), !(!i && o.__c.props === n) && (!f || f.call(this, n, t, r));};}return o.__N || o.__;}function h(u, i) {var o = d(t++, 3);!n.__s && z(o.__H, i) && (o.__ = u, o.i = i, r.__H.__h.push(o));}function s(u, i) {var o = d(t++, 4);!n.__s && z(o.__H, i) && (o.__ = u, o.i = i, r.__h.push(o));}function _(n) {return o = 5, F(function () {return { current: n };}, []);}function A(n, t, r) {o = 6, s(function () {return "function" == typeof n ? (n(t()), function () {return n(null);}) : n ? (n.current = t(), function () {return n.current = null;}) : void 0;}, null == r ? r : r.concat(n));}function F(n, r) {var u = d(t++, 7);return z(u.__H, r) ? (u.__V = n(), u.i = r, u.__h = n, u.__V) : u.__;}function T(n, t) {return o = 8, F(function () {return n;}, t);}function q(n) {var u = r.context[n.__c],i = d(t++, 9);return i.c = n, u ? (null == i.__ && (i.__ = !0, u.sub(r)), u.props.value) : n.__;}function x(t, r) {n.useDebugValue && n.useDebugValue(r ? r(t) : t);}function P(n) {var u = d(t++, 10),i = p();return u.__ = n, r.componentDidCatch || (r.componentDidCatch = function (n, t) {u.__ && u.__(n, t), i[1](n);}), [i[0], function () {i[1](void 0);}];}function V() {var n = d(t++, 11);return n.__ || (n.__ = "P" + function (n) {for (var t = 0, r = n.length; r > 0;) t = (t << 5) - t + n.charCodeAt(--r) | 0;return t;}(r.__v.__m) + t), n.__;}function b() {for (var t; t = f.shift();) if (t.__P && t.__H) try {t.__H.__h.forEach(k), t.__H.__h.forEach(w), t.__H.__h = [];} catch (r) {t.__H.__h = [], n.__e(r, t.__v);}}n.__b = function (n) {"function" != typeof n.type || n.__m || null === n.__ ? n.__m || (n.__m = n.__ && n.__.__m ? n.__.__m : "") : n.__m = (n.__ && n.__.__m ? n.__.__m : "") + (n.__ && n.__.__k ? n.__.__k.indexOf(n) : 0), r = null, e && e(n);}, n.__r = function (n) {a && a(n), t = 0;var i = (r = n.__c).__H;i && (u === r ? (i.__h = [], r.__h = [], i.__.forEach(function (n) {n.__N && (n.__ = n.__N), n.__V = c, n.__N = n.i = void 0;})) : (i.__h.forEach(k), i.__h.forEach(w), i.__h = [])), u = r;}, n.diffed = function (t) {v && v(t);var o = t.__c;o && o.__H && (o.__H.__h.length && (1 !== f.push(o) && i === n.requestAnimationFrame || ((i = n.requestAnimationFrame) || j)(b)), o.__H.__.forEach(function (n) {n.i && (n.__H = n.i), n.__V !== c && (n.__ = n.__V), n.i = void 0, n.__V = c;})), u = r = null;}, n.__c = function (t, r) {r.some(function (t) {try {t.__h.forEach(k), t.__h = t.__h.filter(function (n) {return !n.__ || w(n);});} catch (u) {r.some(function (n) {n.__h && (n.__h = []);}), r = [], n.__e(u, t.__v);}}), l && l(t, r);}, n.unmount = function (t) {m && m(t);var r,u = t.__c;u && u.__H && (u.__H.__.forEach(function (n) {try {k(n);} catch (n) {r = n;}}), u.__H = void 0, r && n.__e(r, u.__v));};var g = "function" == typeof requestAnimationFrame;function j(n) {var t,r = function () {clearTimeout(u), g && cancelAnimationFrame(t), setTimeout(n);},u = setTimeout(r, 100);g && (t = requestAnimationFrame(r));}function k(n) {var t = r,u = n.__c;"function" == typeof u && (n.__c = void 0, u()), r = t;}function w(n) {var t = r;n.__c = n.__(), r = t;}function z(n, t) {return !n || n.length !== t.length || t.some(function (t, r) {return t !== n[r];});}function B(n, t) {return "function" == typeof t ? t(n) : t;} return { useCallback: T, useContext: q, useDebugValue: x, useEffect: h, useErrorBoundary: P, useId: V, useImperativeHandle: A, useLayoutEffect: s, useMemo: F, useReducer: y, useRef: _, useState: p }; })(); -export { useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useState }; +export {useCallback, useContext, useDebugValue, useEffect, useErrorBoundary, useId, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState}; + +// signals-core +const {Signal, batch, computed, effect, signal} = (function() { + function i(){throw new Error("Cycle detected")}function t(){if(!(n>1)){var i,t=!1;while(void 0!==r){var h=r;r=void 0;s++;while(void 0!==h){var o=h.o;h.o=void 0;h.f&=-3;if(!(8&h.f)&&d(h))try{h.c()}catch(h){if(!t){i=h;t=!0}}h=o}}s=0;n--;if(t)throw i}else n--}function h(i){if(n>0)return i();n++;try{return i()}finally{t()}}var o=void 0,r=void 0,n=0,s=0,f=0;function v(i){if(void 0!==o){var t=i.n;if(void 0===t||t.t!==o){o.s=t={i:0,S:i,p:void 0,n:o.s,t:o,e:void 0,x:void 0,r:t};i.n=t;if(32&o.f)i.S(t);return t}else if(-1===t.i){t.i=0;if(void 0!==t.p){t.p.n=t.n;if(void 0!==t.n)t.n.p=t.p;t.p=void 0;t.n=o.s;o.s.p=t;o.s=t}return t}}}function e(i){this.v=i;this.i=0;this.n=void 0;this.t=void 0}e.prototype.h=function(){return!0};e.prototype.S=function(i){if(this.t!==i&&void 0===i.e){i.x=this.t;if(void 0!==this.t)this.t.e=i;this.t=i}};e.prototype.U=function(i){var t=i.e,h=i.x;if(void 0!==t){t.x=h;i.e=void 0}if(void 0!==h){h.e=t;i.x=void 0}if(i===this.t)this.t=h};e.prototype.subscribe=function(i){var t=this;return b(function(){var h=t.value,o=32&this.f;this.f&=-33;try{i(h)}finally{this.f|=o}})};e.prototype.valueOf=function(){return this.value};e.prototype.toString=function(){return this.value+""};e.prototype.peek=function(){return this.v};Object.defineProperty(e.prototype,"value",{get:function(){var i=v(this);if(void 0!==i)i.i=this.i;return this.v},set:function(h){if(h!==this.v){if(s>100)i();this.v=h;this.i++;f++;n++;try{for(var o=this.t;void 0!==o;o=o.x)o.t.N()}finally{t()}}}});function u(i){return new e(i)}function d(i){for(var t=i.s;void 0!==t;t=t.n)if(t.S.i!==t.i||!t.S.h()||t.S.i!==t.i)return!0;return!1}function c(i){for(var t=i.s;void 0!==t;t=t.n){var h=t.S.n;if(void 0!==h)t.r=h;t.S.n=t;t.i=-1}}function a(i){var t=i.s,h=void 0;while(void 0!==t){var o=t.n;if(-1===t.i){t.S.U(t);t.n=void 0}else{if(void 0!==h)h.p=t;t.p=void 0;t.n=h;h=t}t.S.n=t.r;if(void 0!==t.r)t.r=void 0;t=o}i.s=h}function l(i){e.call(this,void 0);this.x=i;this.s=void 0;this.g=f-1;this.f=4}(l.prototype=new e).h=function(){this.f&=-3;if(1&this.f)return!1;if(32==(36&this.f))return!0;this.f&=-5;if(this.g===f)return!0;this.g=f;this.f|=1;if(this.i>0&&!d(this)){this.f&=-2;return!0}var i=o;try{c(this);o=this;var t=this.x();if(16&this.f||this.v!==t||0===this.i){this.v=t;this.f&=-17;this.i++}}catch(i){this.v=i;this.f|=16;this.i++}o=i;a(this);this.f&=-2;return!0};l.prototype.S=function(i){if(void 0===this.t){this.f|=36;for(var t=this.s;void 0!==t;t=t.n)t.S.S(t)}e.prototype.S.call(this,i)};l.prototype.U=function(i){e.prototype.U.call(this,i);if(void 0===this.t){this.f&=-33;for(var t=this.s;void 0!==t;t=t.n)t.S.U(t)}};l.prototype.N=function(){if(!(2&this.f)){this.f|=6;for(var i=this.t;void 0!==i;i=i.x)i.t.N()}};l.prototype.peek=function(){if(!this.h())i();if(16&this.f)throw this.v;return this.v};Object.defineProperty(l.prototype,"value",{get:function(){if(1&this.f)i();var t=v(this);this.h();if(void 0!==t)t.i=this.i;if(16&this.f)throw this.v;return this.v}});function w(i){return new l(i)}function y(i){var h=i.u;i.u=void 0;if("function"==typeof h){n++;var r=o;o=void 0;try{h()}catch(t){i.f&=-2;i.f|=8;_(i);throw t}finally{o=r;t()}}}function _(i){for(var t=i.s;void 0!==t;t=t.n)t.S.U(t);i.x=void 0;i.s=void 0;y(i)}function g(i){if(o!==this)throw new Error("Out-of-order effect");a(this);o=i;this.f&=-2;if(8&this.f)_(this);t()}function p(i){this.x=i;this.u=void 0;this.s=void 0;this.o=void 0;this.f=32}p.prototype.c=function(){var i=this.S();try{if(!(8&this.f)&&void 0!==this.x)this.u=this.x()}finally{i()}};p.prototype.S=function(){if(1&this.f)i();this.f|=1;this.f&=-9;y(this);c(this);n++;var t=o;o=this;return g.bind(this,t)};p.prototype.N=function(){if(!(2&this.f)){this.f|=2;this.o=r;r=this}};p.prototype.d=function(){this.f|=8;if(!(1&this.f))_(this)};function b(i){var t=new p(i);t.c();return t.d.bind(t)} + return {Signal: e, batch: h, computed: w, effect: b, signal: u}; +})(); +export {Signal, batch, computed, effect, signal}; + +// signals +const {useComputed, useSignal, useSignalEffect} = (function() { + var n = Component, i = options, r = useMemo, t = useRef, f = useEffect, o = Signal, e = computed, u = signal, a = effect; + var c,v;function s(n,r){i[n]=r.bind(null,i[n]||function(){})}function l(n){if(v)v();v=n&&n.S()}function p(n){var i=this,t=n.data,f=useSignal(t);f.value=t;var o=r(function(){var n=i.__v;while(n=n.__)if(n.__c){n.__c.__$f|=4;break}i.__$u.c=function(){i.base.data=o.peek()};return e(function(){var n=f.value.value;return 0===n?0:!0===n?"":n||""})},[]);return o.value}p.displayName="_st";Object.defineProperties(o.prototype,{constructor:{configurable:!0},type:{configurable:!0,value:p},props:{configurable:!0,get:function(){return{data:this}}},__b:{configurable:!0,value:1}});s("__b",function(n,i){if("string"==typeof i.type){var r,t=i.props;for(var f in t)if("children"!==f){var e=t[f];if(e instanceof o){if(!r)i.__np=r={};r[f]=e;t[f]=e.peek()}}}n(i)});s("__r",function(n,i){l();var r,t=i.__c;if(t){t.__$f&=-2;if(void 0===(r=t.__$u))t.__$u=r=function(n){var i;a(function(){i=this});i.c=function(){t.__$f|=1;t.setState({})};return i}()}c=t;l(r);n(i)});s("__e",function(n,i,r,t){l();c=void 0;n(i,r,t)});s("diffed",function(n,i){l();c=void 0;var r;if("string"==typeof i.type&&(r=i.__e)){var t=i.__np,f=i.props;if(t){var o=r.U;if(o)for(var e in o){var u=o[e];if(void 0!==u&&!(e in t)){u.d();o[e]=void 0}}else r.U=o={};for(var a in t){var v=o[a],s=t[a];if(void 0===v){v=d(r,a,s,f);o[a]=v}else v.o(s,f)}}}n(i)});function d(n,i,r,t){var f=i in n&&void 0===n.ownerSVGElement,o=u(r);return{o:function(n,i){o.value=n;t=i},d:a(function(){var r=o.value.value;if(t[i]!==r){t[i]=r;if(f)n[i]=r;else if(r)n.setAttribute(i,r);else n.removeAttribute(i)}})}}s("unmount",function(n,i){if("string"==typeof i.type){var r=i.__e;if(r){var t=r.U;if(t){r.U=void 0;for(var f in t){var o=t[f];if(o)o.d()}}}}else{var e=i.__c;if(e){var u=e.__$u;if(u){e.__$u=void 0;u.d()}}}n(i)});s("__h",function(n,i,r,t){if(t<3)i.__$f|=2;n(i,r,t)});n.prototype.shouldComponentUpdate=function(n,i){var r=this.__$u;if(!(r&&void 0!==r.s||4&this.__$f))return!0;if(3&this.__$f)return!0;for(var t in i)return!0;for(var f in n)if("__source"!==f&&n[f]!==this.props[f])return!0;for(var o in this.props)if(!(o in n))return!0;return!1};function useSignal(n){return r(function(){return u(n)},[])}function useComputed(n){var i=t(n);i.current=n;c.__$f|=4;return r(function(){return e(function(){return i.current()})},[])}function useSignalEffect(n){var i=t(n);i.current=n;f(function(){return a(function(){i.current()})},[])} + return {useComputed, useSignal, useSignalEffect} +})(); +export {useComputed, useSignal, useSignalEffect}; diff --git a/static/search-suggestions.js b/static/search-suggestions.js index b3d08040..1a9a3d3e 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -1,17 +1,20 @@ -import {h, htm, render, useState, useEffect, createContext, useContext} from "./preact.js" +import {h, htm, render, useState, useEffect, createContext, useContext, signal, computed, effect} from "./preact.js" const html = htm.bind(h) const classNames = classArr => classArr.filter(el => el).join(" ") -const form = document.getElementById("bw-pr-search") +const eForm = document.getElementById("bw-pr-search-form") +const eInput = document.getElementById("bw-pr-search-input") +const eSuggestions = document.getElementById("bw-pr-search-suggestions") -const AcceptSuggestion = createContext(null) const hitsPromise = new Map() const hitsDone = new Set() -function Suggestion(props) { - const acceptSuggestion = useContext(AcceptSuggestion) - return html`
  • ` -} +const query = signal("") +const focus = signal(false) +const st = signal("ready") +const suggestions = signal([]) + +// processing functions function fetchSuggestions(query, setSuggestions) { if (query === "") query = "\0" @@ -36,58 +39,53 @@ function fetchSuggestions(query, setSuggestions) { return promise } -function SuggestionList(props) { - return html` -
    -
      - ${props.hits.map(hit => html`<${Suggestion} ...${hit} />`)} -
    -
    ` +function acceptSuggestion(hit) { + st.value = "accepted" + query.value = hit.title + const dest = new URL(hit.url).pathname.match("/wiki/.*") + location = `/${BWData.wikiname}${dest}` } -function ControlledInput() { - const [query, setQuery] = useState("") - const [focus, setFocus] = useState(false) - const [st, setSt] = useState("ready") - const [suggestions, setSuggestions] = useState([]) +// suggestion list view - useEffect(() => { - if (st === "accepted") return - setSt("loading") - fetchSuggestions(query).then(s => { - setSuggestions(s) - if (hitsDone.size === hitsPromise.size) { - setSt("ready") - } - }) - }, [query]) +function Suggestion(hit) { + return html`
  • ` +} - function acceptSuggestion(suggestion) { - setQuery(suggestion.title) - setSt("accepted") - const dest = new URL(suggestion.url).pathname.match("/wiki/.*") - location = `/${BWData.wikiname}${dest}` - } +function SuggestionList() { + return html` +
      + ${suggestions.value.map(hit => html`<${Suggestion} ...${hit} />`)} +
    ` +} - useEffect(() => { - function listener(event) { - if (event.type === "focusin") setFocus(true) - else setFocus(false) - } - form.addEventListener("focusin", listener) - form.addEventListener("focusout", listener) - return () => { - form.removeEventListener("focusin", listener) - form.removeEventListener("focusout", listener) +render(html`<${SuggestionList} />`, eSuggestions) + +// input view + +effect(() => { + if (st.peek() === "accepted") return // lock results from changing during navigation + st.value = "loading" + fetchSuggestions(query.value).then(res => { + suggestions.value = res + if (hitsDone.size === hitsPromise.size) { + st.value = "ready" } }) +}) +document.addEventListener("pageshow", () => { + st.value = "ready" // unlock results from changing after returning to page +}) + +function SuggestionInput() { return html` -<${AcceptSuggestion.Provider} value=${acceptSuggestion}> - - setQuery(e.target.value)} value=${query} class=${classNames(["bw-ss__input", `bw-ss__input--${st}`])} /> - <${SuggestionList} hits=${suggestions} focus=${focus} st=${st}/> - query.value = e.target.value} value=${query.value} class=${classNames(["bw-ss__input", `bw-ss__input--${st.value}`])} />` } -render(html`<${ControlledInput} />`, form) +render(html`<${SuggestionInput} />`, eInput) + +// form focus + +eForm.addEventListener("focusin", () => focus.value = true) +eForm.addEventListener("focusout", () => focus.value = false) From 8b200d621ab6fa4081ba81fe6bd1eed66fd6a4b5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 30 Oct 2022 23:15:26 +1300 Subject: [PATCH 058/166] Link out to NIWA's wikis where available --- src/application-globals.rkt | 35 +++++++- src/niwa-data.rkt | 156 ++++++++++++++++++++++++++++++++++++ src/page-home.rkt | 13 +-- static/main.css | 59 ++++++++++++++ 4 files changed, 256 insertions(+), 7 deletions(-) create mode 100644 src/niwa-data.rkt diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 91c7e5a0..55dfddcc 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -1,11 +1,14 @@ #lang racket/base -(require racket/string +(require racket/list + racket/string json (prefix-in easy: net/http-easy) html-writing web-server/http "config.rkt" "data.rkt" + "niwa-data.rkt" + "pure-utils.rkt" "xexpr-utils.rkt" "url-utils.rkt") @@ -62,6 +65,35 @@ " Media files and official Fandom documents have different copying restrictions.") (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))))))) +;; generate a notice with a link if a fandom wiki has a replacement as part of NIWA or similar +;; if the wiki has no replacement, display nothing +(define (niwa-notice wikiname title) + (define ind (findf (λ (item) (member wikiname (first item))) niwa-data)) + (if ind + (let* ([search-page (format "/Special:Search?~a" + (params->query `(("search" . ,title) + ("go" . "Go"))))] + [go (if (string-suffix? (third ind) "/") + (regexp-replace "/$" (third ind) (λ (_) search-page)) + (let* ([_ (println (regexp-match "/(w[^./]*)/" (third ind)))] [joiner (second (regexp-match "/(w[^./]*)/" (third ind)))]) + (regexp-replace "/w[^./]*/.*$" (third ind) (λ (_) (format "/~a~a" joiner search-page)))))]) + `(aside (@ (class "niwa__notice")) + (h1 (@ (class "niwa__header")) ,(second ind) " has its own website separate from Fandom.") + (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,(second ind) " →") + (div (@ (class "niwa__cols")) + (div (@ (class "niwa__left")) + (p "Most major Nintendo wikis are part of the " + (a (@ (href "https://www.niwanetwork.org/about/")) "Nintendo Independent Wiki Alliance") + " and have their own wikis off Fandom. You can help this wiki by " + (a (@ (href ,go)) "visiting it directly.")) + (p ,(fifth ind)) + (div (@ (class "niwa__divider"))) + (p "Why are you seeing this message? Fandom refuses to delete or archive their copy of this wiki, so that means their pages will appear high up in search results. Fandom hopes to get clicks from readers who don't know any better.") + (p (@ (class "niwa__feedback")) (a (@ (href "https://www.kotaku.com.au/2022/10/massive-zelda-wiki-reclaims-independence-six-months-before-tears-of-the-kingdom/")) "More info") " / " (a (@ (href "https://docs.breezewiki.com/Reporting_Bugs.html")) "Feedback on this notice?"))) + (div (@ (class "niwa__right")) + (img (@ (class "niwa__logo") (src ,(format "https://www.niwanetwork.org~a" (fourth ind))))))))) + "")) + (define (generate-wiki-page content #:source-url source-url @@ -107,6 +139,7 @@ (div (@ (class "fandom-community-header__background tileHorizontally header"))) (div (@ (class "page")) (main (@ (class "page__main")) + ,(niwa-notice wikiname title) (div (@ (class "custom-top")) (h1 (@ (class "page-title")) ,title) (nav (@ (class "sitesearch")) diff --git a/src/niwa-data.rkt b/src/niwa-data.rkt new file mode 100644 index 00000000..24279d6d --- /dev/null +++ b/src/niwa-data.rkt @@ -0,0 +1,156 @@ +#lang racket/base + +(provide + niwa-data) + +;; wikiname, niwa-name, url, logo-url +(define niwa-data + '((("arms" "armsgame") + "ARMS Institute" + "https://armswiki.org/wiki/Home" + "/images/logos/armswiki.png" + "ARMS Institute is a comprehensive resource for information about the Nintendo Switch game, ARMS. Founded on May 1, 2017 and growing rapidly, the wiki strives to offer in-depth coverage of ARMS from both a competitive and casual perspective. Join us and ARM yourself with knowledge!") + (("pokemon" "monster") + "Bulbapedia" + "https://bulbapedia.bulbagarden.net/wiki/Main_Page" + "/images/logos/bulbapedia.png" + "A part of the Bulbagarden community, Bulbapedia was founded on December 21, 2004 by Liam Pomfret. Everything you need to know about Pokémon can be found at Bulbapedia, whether about the games, the anime, the manga, or something else entirely. With its Bulbanews section and the Bulbagarden forums, it's your one-stop online place for Pokémon.") + (("dragalialost") + "Dragalia Lost Wiki" + "https://dragalialost.wiki/w/Dragalia_Lost_Wiki" + "/images/logos/dragalialost.png" + "The Dragalia Lost Wiki was originally founded in September 2018 on the Gamepedia platform but went independent in January 2021. The Wiki aims to document anything and everything Dragalia Lost, from in-game data to mechanics, story, guides, and more!") + (("dragonquest") + "Dragon Quest Wiki" + "https://dragon-quest.org/wiki/Main_Page" + "/images/logos/dragonquestwiki.png" + "Originally founded on Wikia, the Dragon Quest Wiki was largely inactive until FlyingRagnar became an admin in late 2009. The wiki went independent about a year later when it merged with the Dragon Quest Dictionary/Encyclopedia which was run by Zenithian and supported by the Dragon's Den. The Dragon Quest Wiki aims to be the most complete resource for Dragon Quest information on the web. It continues to grow in the hope that one day the series will be as popular in the rest of the world as it is in Japan.") + (("fireemblem") + "Fire Emblem Wiki" + "https://fireemblemwiki.org/wiki/Main_Page" + "/images/logos/fireemblemwiki.png" + "Growing since August 26, 2010, Fire Emblem Wiki is a project whose goal is to cover all information pertaining to the Fire Emblem series. It aspires to become the most complete and accurate independent source of information on this series.") + (("fzero" "f-zero") + "F-Zero Wiki" + "https://mutecity.org/wiki/F-Zero_Wiki" + "/images/logos/fzerowiki.png" + "Founded on Wikia in November 2007, F-Zero Wiki became independent with NIWA's help in 2011. F-Zero Wiki is quickly growing into the Internet's definitive source for the world of 2200 km/h+, from pilots to machines, and is the founding part of MuteCity.org, the web's first major F-Zero community.") + (("goldensun") + "Golden Sun Universe" + "https://www.goldensunwiki.net/wiki/Main_Page" + "/images/logos/goldensununiverse.png" + "Originally founded on Wikia in late 2006, Golden Sun Universe has always worked hard to meet one particular goal: to be the single most comprehensive yet accessible resource on the Internet for Nintendo's RPG series Golden Sun. It became an independent wiki four years later. Covering characters and plot, documenting all aspects of the gameplay, featuring walkthroughs both thorough and bare-bones, and packed with all manner of odd and fascinating minutiae, Golden Sun Universe leaves no stone unturned!") + (("tetris") + "Hard Drop - Tetris Wiki" + "https://harddrop.com/wiki/Main_Page" + "/images/logos/harddrop.png" + "The Tetris Wiki was founded by Tetris fans for Tetris fans on tetrisconcept.com in March 2006. The Tetris Wiki torch was passed to harddrop.com in July 2009. Hard Drop is a Tetris community for all Tetris players, regardless of skill or what version of Tetris you play.") + (("kidicarus") + "Icaruspedia" + "https://www.kidicaruswiki.org/wiki/Main_Page" + "/images/logos/icaruspedia.png" + "Icaruspedia is the Kid Icarus wiki that keeps flying to new heights. After going independent on January 8, 2012, Icaruspedia has worked to become the largest and most trusted independent source of Kid Icarus information. Just like Pit, they\"ll keep on fighting until the job is done.") + (("splatoon" "uk-splatoon" "splatoon3" "splatoon2") + "Inkipedia" + "https://splatoonwiki.org/wiki/Main_Page" + "/images/logos/inkipedia.png" + "Inkipedia is your ever-growing go-to source for all things Splatoon related. Though founded on Wikia on June 10, 2014, Inkipedia went independent on May 18, 2015, just days before Splatoon's release. Our aim is to cover all aspects of the series, both high and low. Come splat with us now!") + (("starfox") + "Lylat Wiki" + "https://starfoxwiki.info/wiki/Lylat_Wiki" + "/images/logos/lylatwiki.png" + "Out of seemingly nowhere, Lylat Wiki sprung up one day in early 2010. Led by creator, Justin Folvarcik, and project head, Tacopill, the wiki has reached stability since the move to its own domain. The staff of Lylat Wiki are glad to help out the NIWA wikis and are even prouder to join NIWA's ranks as the source for information on the Star Fox series.") + (("metroid" "themetroid") + "Metroid Wiki" + "https://www.metroidwiki.org/wiki/Main_Page" + "/images/logos/metroidwiki.png" + "Metroid Wiki, founded on January 27, 2010 by Nathanial Rumphol-Janc and Zelda Informer, is a rapidly expanding wiki that covers everything Metroid, from the games, to every suit, vehicle and weapon.") + (("nintendo" "nintendoseries" "nintendogames") + "Nintendo Wiki" + "http://niwanetwork.org/wiki/Main_Page" + "/images/logos/nintendowiki.png" + "Created on May 12, 2010, NintendoWiki (N-Wiki) is a collaborative project by the NIWA team to create an encyclopedia dedicated to Nintendo, being the company around which all other NIWA content is focused. It ranges from mainstream information such as the games and people who work for the company, to the most obscure info like patents and interesting trivia.") + (("animalcrossing" "animalcrossingcf" "acnh") + "Nookipedia" + "https://nookipedia.com/wiki/Main_Page" + "/images/logos/nookipedia.png" + "Founded in August 2005 on Wikia, Nookipedia was originally known as Animal Crossing City. Shortly after its five-year anniversary, Animal Crossing City decided to merge with the independent Animal Crossing Wiki, which in January 2011 was renamed to Nookipedia. Covering everything from the series including characters, items, critters, and much more, Nookipedia is your number one resource for everything Animal Crossing!") + (("pikmin") + "Pikipedia" + "https://www.pikminwiki.com/" + "/images/logos/pikipedia.png" + "Pikipedia, also known as Pikmin Wiki, was founded by Dark Lord Revan on Wikia in December 2005. In September 2010, with NIWA's help, Pikipedia moved away from Wikia to become independent. Pikipedia is working towards their goal of being the foremost source for everything Pikmin.") + (("pikmin-fan" "pikpikpedia") + "Pimkin Fanon" + "https://www.pikminfanon.com/wiki/Main_Page" + "/images/logos/pikifanon.png" + "Pikmin Fanon is a Pikmin wiki for fan stories (fanon). Founded back on November 1, 2008 by Rocky0718 as a part of Wikia, Pikmin Fanon has been independent since September 14, 2010. Check them out for fan created stories based around the Pikmin series.") + (("supersmashbros") + "SmashWiki" + "https://www.ssbwiki.com/" + "/images/logos/smashwiki.png" + "Originally two separate wikis (one on SmashBoards, the other on Wikia), SmashWiki as we know it was formed out of a merge on February 29th, 2008, becoming independent on September 28th, 2010. SmashWiki is the premier source of Smash Bros. information, from simple tidbits to detailed mechanics, and also touches on the origins of its wealth of content from its sibling franchises.") + (("starfy") + "Starfy Wiki" + "https://www.starfywiki.org/wiki/Main_Page" + "/images/logos/starfywiki.png" + "Founded on May 30, 2009, Starfy Wiki's one goal is to become the best source on Nintendo's elusive game series The Legendary Starfy. After gaining independence in 2011 with the help of Tappy and the wiki's original administrative team, the wiki still hopes to achieve its goal and be the best source of Starfy info for all present and future fans.") + (() + "StrategyWiki" + "https://www.strategywiki.org/wiki/Main_Page" + "/images/logos/strategywiki.png" + "StrategyWiki was founded in December 2005 by former member Brandon Suit with the idea that the existing strategy guides on the Internet could be improved. Three years later, in December 2008, Scott Jacobi officially established Abxy LLC for the purpose of owning and operating StrategyWiki as a community. Their vision is to bring free, collaborative video game strategy guides to the masses, including Nintendo franchise strategy guides.") + (("mario" "themario" "imario" "supermarionintendo" "mariokart" "luigi-kart" "mario3") + "Super Mario Wiki" + "https://www.mariowiki.com/" + "/images/logos/mariowiki.png" + "Online since August 12, 2005, when it was founded by Steve Shinn, Super Mario Wiki has you covered for anything Mario, Donkey Kong, Wario, Luigi, Yoshi—the whole gang, in fact. With its own large community in its accompanying forum, Super Mario Wiki is not only a great encyclopedia, but a fansite for you to talk anything Mario.") + (("mario64") + "Ukikipedia" + "https://ukikipedia.net/wiki/Main_Page" + "/images/logos/ukikipedia.png" + "Founded in 2018, Ukikipedia is a wiki focused on expert level knowledge of Super Mario 64, including detailed coverage of game mechanics, glitches, speedrunning, and challenges.") + (("advancewars") + "Wars Wiki" + "https://www.warswiki.org/wiki/Main_Page" + "/images/logos/warswiki.png" + "Created in February 2009, Wars Wiki is a small wiki community with a large heart. Founded by JoJo and Wars Central, Wars Wiki is going strong on one of Nintendo's lesser known franchises. Wars Wiki is keen to contribute to NIWA, and we're proud to be able to support them. With the Wars Central community, including forums, it's definitely worth checking out.") + (("earthbound") + "WikiBound" + "https://www.wikibound.info/wiki/WikiBound" + "/images/logos/wikibound.png" + "Founded in early 2010 by Tacopill, WikiBound strives to create a detailed database on the Mother/EarthBound games, a quaint series only having two games officially released outside of Japan. Help spread the PK Love by editing WikiBound!") + (("kirby") + "WiKirby" + "https://wikirby.com/wiki/Kirby_Wiki" + "/images/logos/wikirby.png" + "WiKirby. It's a wiki. About Kirby! Amidst the excitement of NIWA being founded, Josh LeJeune decided to create a Kirby Wiki, due to lack of a strong independent one online. Coming online on January 24, 2010, WiKirby continues its strong launch with a dedicated community and a daily growing source of Kirby based knowledge.") + (("xenoblade" "xenoseries" "xenogears" "xenosaga") + "Xeno Series Wiki" + "https://www.xenoserieswiki.org/wiki/Main_Page" + "/images/logos/xenoserieswiki.png" + "Xeno Series Wiki was created February 4, 2020 by Sir Teatei Moonlight. While founded by the desire to have an independent wiki for Xenoblade, there was an interest in including the Xenogears and Xenosaga games within its focus as well. This wide range of coverage means it's always in need of new editors to help bolster its many subjects.") + (("zelda" "zelda-archive") + "Zeldapedia" + "https://zeldapedia.wiki/wiki/Main_Page" + "/images/logos/zeldapedia.png" + "Founded on April 23, 2005 as Zelda Wiki, today's Zeldapedia is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news."))) + +;; get the current dataset so it can be stored above +(module+ test + (require racket/generator + racket/list + net/http-easy + html-parsing + "xexpr-utils.rkt") + (define r (get "https://www.niwanetwork.org/members/")) + (define x (html->xexp (bytes->string/utf-8 (response-body r)))) + (define english ((query-selector (λ (e a c) (println a) (equal? (get-attribute 'id a) "content1")) x))) + (define gen (query-selector (λ (e a c) (has-class? "member" a)) english)) + (for/list ([item (in-producer gen #f)]) + (define links (query-selector (λ (e a c) (eq? e 'a)) item)) + (define url (get-attribute 'href (bits->attributes (links)))) + (define title (third (links))) + (define icon (get-attribute 'src (bits->attributes ((query-selector (λ (e a c) (eq? e 'img)) item))))) + (define description (second ((query-selector (λ (e a c) (eq? e 'p)) item)))) + (list '() title url icon description))) diff --git a/src/page-home.rkt b/src/page-home.rkt index 86c316d1..9e7c954a 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -15,13 +15,11 @@ (require rackunit)) (define examples - '(("crosscode" "CrossCode_Wiki") - ("pokemon" "Eevee") - ("minecraft" "Bricks") + '(("minecraft" "Bricks") + ("crosscode" "CrossCode_Wiki") ("undertale" "Hot_Dog...%3F") ("tardis" "Eleanor_Blake") - ("fireemblem" "God-Shattering_Star") - ("fallout" "Pip-Boy_3000"))) + ("zelda" "Boomerang"))) (define content `((h2 "BreezeWiki makes wiki pages on Fandom readable") @@ -53,6 +51,8 @@ (p (@ (class "testimonial")) ">you are so right that fandom still sucks even with adblock somehow. even zapping all the stupid padding it still sucks —Minimus") (p (@ (class "testimonial")) ">attempting to go to a wiki's forum page with breezewiki doesn't work, which is based honestly —Tom Skeleton") (p (@ (class "testimonial")) ">Fandom pages crashing and closing, taking forever to load and locking up as they load the ads on the site... they are causing the site to crash because they are trying to load video ads both at the top and bottom of the site as well as two or three banner ads, then a massive top of site ad and eventually my anti-virus shuts the whole site down because it's literally pulling more resources than WoW in ultra settings... —Anonymous") + (p (@ (class "testimonial")) ">reblogs EXTREMELY appreciated I want that twink* (*fandom wiki) obliterated —footlong") + (h2 "What BreezeWiki isn't") (p "BreezeWiki isn't an \"alternative\" to Fandom, and it doesn't let you edit or write new pages.") (p "If you want to create your own wiki, try Miraheze!"))) @@ -63,7 +63,8 @@ (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) (title "About | BreezeWiki") (link (@ (rel "stylesheet") (type "text/css") (href "/static/internal.css"))) - (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css")))) + (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css"))) + (link (@ (rel "icon") (href "/static/breezewiki-icon.svg")))) (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileBoth header"))) diff --git a/static/main.css b/static/main.css index c4b6c209..d0bfe8cb 100644 --- a/static/main.css +++ b/static/main.css @@ -285,6 +285,61 @@ figcaption, .lightbox-caption, .thumbcaption { text-align: left; } +/* nintendo independent wiki alliance notice */ +.niwa__notice { + background: #fdedd8; + color: black; + border: 1px dashed black; + padding: 3vw; + margin-bottom: 3vw; + border-radius: 6px; + font-size: 18px; +} +.niwa__header { + font-size: max(2.9vw, 26px); + margin-top: 0; +} +.niwa__notice a { + color: #002263; + text-decoration: underline; +} +.niwa__notice .niwa__go { + display: inline-block; + border-radius: 20px; + padding: 16px 26px; + background: #f2f65f; + color: black; + text-decoration: none; + font-size: 24px; + font-weight: bold; + line-height: 1.2; + border: 2px solid black; + box-shadow: 0 5px 0 black; + margin-bottom: 8px; +} +.niwa__notice .niwa__go:hover { + color: black; + text-decoration: underline; + background: #dee154; +} +.niwa__cols { + display: grid; + grid-template-columns: 1fr auto; + gap: 8px; +} +.niwa__logo { + width: 150px; + height: auto; +} +.niwa__divider { + height: 1px; + background: #808080; +} +.niwa__feedback { + font-size: 14px; + text-align: right; +} + /* media queries */ /* for reference, cell phone screens are generally 400 px wide, definitely less than 500 px */ @@ -301,6 +356,10 @@ figcaption, .lightbox-caption, .thumbcaption { body.skin-fandomdesktop, button, input, textarea, .wikitable, .va-table { font-size: 16px; } + /* niwa layout */ + .niwa__right { + display: none; + } } @media (min-width: 560px) { /* wider than 560 px */ From d3c5498d47869b3623b88dece68048d31867a2b8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 30 Oct 2022 23:51:15 +1300 Subject: [PATCH 059/166] "Cache-busting" for static files --- src/application-globals.rkt | 10 +++++----- src/niwa-data.rkt | 4 ++-- src/page-home.rkt | 7 ++++--- src/page-static.rkt | 4 +++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 55dfddcc..bbd52360 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -8,6 +8,7 @@ "config.rkt" "data.rkt" "niwa-data.rkt" + "static-data.rkt" "pure-utils.rkt" "xexpr-utils.rkt" "url-utils.rkt") @@ -37,7 +38,7 @@ `(footer (@ (class "custom-footer")) (div (@ (class ,(if source-url "custom-footer__cols" "internal-footer"))) (div (p - (img (@ (class "my-logo") (src "/static/breezewiki.svg")))) + (img (@ (class "my-logo") (src ,(get-static-url "breezewiki.svg"))))) (p (a (@ (href "https://gitdab.com/cadence/breezewiki")) ,(format "~a source code" (config-get 'application_name)))) @@ -127,14 +128,13 @@ ,@(map (λ (url) `(link (@ (rel "stylesheet") (type "text/css") (href ,url)))) (required-styles (format "https://~a.fandom.com" wikiname))) - (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css"))) + (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) (script "const BWData = " ,(jsexpr->string (hasheq 'wikiname wikiname 'strict_proxy (config-true? 'strict_proxy)))) ,(if (config-true? 'feature_search_suggestions) - '(script (@ (type "module") (src "/static/search-suggestions.js"))) - "")) - (body (@ (class ,body-class)) + `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js")))) + "") (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileHorizontally header"))) (div (@ (class "page")) diff --git a/src/niwa-data.rkt b/src/niwa-data.rkt index 24279d6d..1b2d20c7 100644 --- a/src/niwa-data.rkt +++ b/src/niwa-data.rkt @@ -49,7 +49,7 @@ "Icaruspedia" "https://www.kidicaruswiki.org/wiki/Main_Page" "/images/logos/icaruspedia.png" - "Icaruspedia is the Kid Icarus wiki that keeps flying to new heights. After going independent on January 8, 2012, Icaruspedia has worked to become the largest and most trusted independent source of Kid Icarus information. Just like Pit, they\"ll keep on fighting until the job is done.") + "Icaruspedia is the Kid Icarus wiki that keeps flying to new heights. After going independent on January 8, 2012, Icaruspedia has worked to become the largest and most trusted independent source of Kid Icarus information. Just like Pit, they'll keep on fighting until the job is done.") (("splatoon" "uk-splatoon" "splatoon3" "splatoon2") "Inkipedia" "https://splatoonwiki.org/wiki/Main_Page" @@ -145,7 +145,7 @@ "xexpr-utils.rkt") (define r (get "https://www.niwanetwork.org/members/")) (define x (html->xexp (bytes->string/utf-8 (response-body r)))) - (define english ((query-selector (λ (e a c) (println a) (equal? (get-attribute 'id a) "content1")) x))) + (define english ((query-selector (λ (e a c) (equal? (get-attribute 'id a) "content1")) x))) (define gen (query-selector (λ (e a c) (has-class? "member" a)) english)) (for/list ([item (in-producer gen #f)]) (define links (query-selector (λ (e a c) (eq? e 'a)) item)) diff --git a/src/page-home.rkt b/src/page-home.rkt index 9e7c954a..0387fd36 100644 --- a/src/page-home.rkt +++ b/src/page-home.rkt @@ -4,6 +4,7 @@ html-writing web-server/http "application-globals.rkt" + "static-data.rkt" "url-utils.rkt" "xexpr-utils.rkt" "config.rkt") @@ -62,9 +63,9 @@ (head (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) (title "About | BreezeWiki") - (link (@ (rel "stylesheet") (type "text/css") (href "/static/internal.css"))) - (link (@ (rel "stylesheet") (type "text/css") (href "/static/main.css"))) - (link (@ (rel "icon") (href "/static/breezewiki-icon.svg")))) + (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css")))) + (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) + (link (@ (rel "icon") (href ,(get-static-url "breezewiki-icon.svg"))))) (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileBoth header"))) diff --git a/src/page-static.rkt b/src/page-static.rkt index 353a0f86..c9967ad1 100644 --- a/src/page-static.rkt +++ b/src/page-static.rkt @@ -64,5 +64,7 @@ ((files:make #:url->path (lambda (u) ((make-url->path path-static) u)) #:path->mime-type (lambda (u) (ext->mime-type (path-get-extension u))) - #:cache-no-cache (config-true? 'debug) #;"browser applies heuristics if unset") + #:cache-no-cache (config-true? 'debug) + #:cache-immutable (not (config-true? 'debug)) + #:cache-max-age (if (config-true? 'debug) #f 604800)) conn new-req)) From 63d37d5e4f737815310187b34c132ac87d5b857d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 30 Oct 2022 23:56:25 +1300 Subject: [PATCH 060/166] Fix: "Cache-busting" for static files --- src/static-data.rkt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/static-data.rkt diff --git a/src/static-data.rkt b/src/static-data.rkt new file mode 100644 index 00000000..b7fc71ae --- /dev/null +++ b/src/static-data.rkt @@ -0,0 +1,18 @@ +#lang typed/racket/base +(require racket/path + racket/runtime-path) + +(provide + get-static-url) + +(define-runtime-path path-static "../static") + +(define static-data + (for/hash ([f (directory-list path-static)]) : (Immutable-HashTable Path Nonnegative-Integer) + (define built (simple-form-path (build-path path-static f))) + (values built (file-or-directory-modify-seconds built)))) + +(: get-static-url ((U String Path) -> String)) +(define (get-static-url path-or-filename) + (define the-path (simple-form-path (if (path? path-or-filename) path-or-filename (build-path path-static path-or-filename)))) + (format "/static/~a?t=~a" (file-name-from-path the-path) (hash-ref static-data the-path))) From 15b41c24f7dfb55ee243e63a0972b9459845ad64 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 31 Oct 2022 00:00:07 +1300 Subject: [PATCH 061/166] Add breezewiki-icon.svg --- static/breezewiki-icon.svg | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 static/breezewiki-icon.svg diff --git a/static/breezewiki-icon.svg b/static/breezewiki-icon.svg new file mode 100644 index 00000000..5249b759 --- /dev/null +++ b/static/breezewiki-icon.svg @@ -0,0 +1,83 @@ + + + + + + image/svg+xml + + + + + + + + +
    +
    +
    +