From 443f1eecbc4c15c8038920027e62fccbdcb0bbe7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 23 Oct 2024 22:52:00 +1300 Subject: [PATCH 01/30] Add user agent and detect blocked pages --- src/fandom-request.rkt | 40 +++++++++++++++++++++++++++++++++------- src/page-wiki.rkt | 28 ++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/fandom-request.rkt b/src/fandom-request.rkt index 966eeee..c306b04 100644 --- a/src/fandom-request.rkt +++ b/src/fandom-request.rkt @@ -1,12 +1,16 @@ #lang typed/racket/base -(require "config.rkt" +(require racket/format + racket/string + "config.rkt" "../lib/url-utils.rkt") (define-type Headers (HashTable Symbol (U Bytes String))) (require/typed net/http-easy [#:opaque Timeout-Config timeout-config?] [#:opaque Response response?] [#:opaque Session session?] + [response-status-code (Response -> Natural)] [current-session (Parameter Session)] + [current-user-agent (Parameter (U Bytes String))] [make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)] [get ((U Bytes String) [#:close? Boolean] @@ -22,19 +26,41 @@ fandom-get-api timeouts) +(unless (string-contains? (~a (current-user-agent)) "BreezeWiki") + (current-user-agent + (format "BreezeWiki/1.0 (~a) ~a" + (if (config-true? 'canonical_origin) + (config-get 'canonical_origin) + "local") + (current-user-agent)))) + (define timeouts (make-timeout-config #:lease 5 #:connect 5)) +(: last-failure Flonum) +(define last-failure 0.0) +(: stored-failure (Option Response)) +(define stored-failure #f) +(define failure-persist-time 30000) + (: no-headers Headers) (define no-headers '#hasheq()) (: fandom-get (String String [#:headers (Option Headers)] -> Response)) (define (fandom-get wikiname path #:headers [headers #f]) - (define dest-url (string-append "https://www.fandom.com" path)) - (define host (string-append wikiname ".fandom.com")) - (log-outgoing wikiname path) - (get dest-url - #:timeouts timeouts - #:headers (hash-set (or headers no-headers) 'Host host))) + (or + (and ((current-inexact-milliseconds) . < . (+ last-failure failure-persist-time)) stored-failure) + (let () + (define dest-url (string-append "https://www.fandom.com" path)) + (define host (string-append wikiname ".fandom.com")) + (log-outgoing wikiname path) + (define res + (get dest-url + #:timeouts timeouts + #:headers (hash-set (or headers no-headers) 'Host host))) + (when (memq (response-status-code res) '(403 406)) + (set! last-failure (current-inexact-milliseconds)) + (set! stored-failure res)) + res))) (: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response)) (define (fandom-get-api wikiname params #:headers [headers #f]) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index f16792c..da63617 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -18,6 +18,7 @@ "config.rkt" "data.rkt" "fandom-request.rkt" + "../lib/archive-file-mappings.rkt" "../lib/pure-utils.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" @@ -37,8 +38,9 @@ (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) (define user-cookies (user-cookies-getter req)) - (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) + (define path (string-join (cdr segments) "/")) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define-values (dest-res siteinfo) @@ -101,9 +103,31 @@ (write-html body out))))))] [(eq? 404 (easy:response-status-code dest-res)) (next-dispatcher)] + [(memq (easy:response-status-code dest-res) '(403 406)) + (response-handler + (define body + (generate-wiki-page + `(div + (p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.") + (p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.") + (p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom.")))) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title (url-segments->guess-title segments) + #:siteinfo siteinfo)) + (response/output + #:code 503 + #:headers (build-headers + always-headers + (header #"Retry-After" #"30") + (header #"Cache-Control" #"max-age=30, public") + (header #"Refresh" #"35")) + (λ (out) + (write-html body out))))] [else (response-handler - (error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" + (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" wikiname path (easy:response-status-code dest-res) From 23a201cc841ef6a8cc0c06a5a6c3a19b77177aad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Nov 2025 23:06:55 +1300 Subject: [PATCH 02/30] Add JSONP mode and captcha JSONP mode is on by default. It will fetch main wiki pages in the browser, without the server needing to make any requests. To turn it off, add [feature_json] enabled = false to config.ini. Captcha is off by default. It is a custom solution and is still experimental at this stage. If you turn it on, please monitor the logs to see how it goes! config.ini options are as follows: [captcha] enabled = true|false log = true|false ip_header =
--- breezewiki.rkt | 15 +++- lib/tree-updater.rkt | 10 +++ src/config.rkt | 6 ++ src/data.rkt | 12 ++- src/dispatcher-tree.rkt | 10 +++ src/page-captcha.rkt | 177 ++++++++++++++++++++++++++++++++++++++++ src/page-wiki-jsonp.rkt | 68 +++++++++++++++ src/page-wiki.rkt | 98 ++++++++++++---------- static/captcha.js | 24 ++++++ static/jsonp.js | 57 +++++++++++++ static/robots.txt | 1 + 11 files changed, 431 insertions(+), 47 deletions(-) create mode 100644 src/page-captcha.rkt create mode 100644 src/page-wiki-jsonp.rkt create mode 100644 static/captcha.js create mode 100644 static/jsonp.js diff --git a/breezewiki.rkt b/breezewiki.rkt index 5fd34b2..17b619b 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -1,5 +1,7 @@ #lang racket/base -(require web-server/servlet-dispatch +(require racket/splicing + web-server/servlet-dispatch + web-server/safety-limits "src/config.rkt" "src/dispatcher-tree.rkt" "src/reloadable.rkt") @@ -9,6 +11,9 @@ (reloadable-entry-point->procedure (make-reloadable-entry-point (quote varname) filename)))) +(require-reloadable "src/page-captcha.rkt" page-captcha) +(require-reloadable "src/page-captcha.rkt" page-captcha-image) +(require-reloadable "src/page-captcha.rkt" page-captcha-verify) (require-reloadable "src/page-category.rkt" page-category) (require-reloadable "src/page-global-search.rkt" page-global-search) (require-reloadable "src/page-home.rkt" page-home) @@ -22,7 +27,9 @@ (require-reloadable "src/page-static-archive.rkt" page-static-archive) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) +(require-reloadable "src/page-wiki.rkt" page-wiki-with-data) (require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline) +(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp) (require-reloadable "src/page-file.rkt" page-file) (reload!) @@ -34,10 +41,14 @@ (if (config-true? 'debug) "127.0.0.1" #f) (config-get 'bind_host)) #:port (string->number (config-get 'port)) + #:safety-limits (make-safety-limits #:max-request-body-length (* 8 1024 1024)) (λ (quit) (channel-put ch (lambda () (semaphore-post quit))) (dispatcher-tree ; order of these does not matter + page-captcha + page-captcha-image + page-captcha-verify page-category page-global-search page-home @@ -48,7 +59,9 @@ page-set-user-settings page-static-archive page-wiki + page-wiki-with-data page-wiki-offline + page-wiki-jsonp page-file redirect-wiki-home static-dispatcher diff --git a/lib/tree-updater.rkt b/lib/tree-updater.rkt index 109c875..2abf56c 100644 --- a/lib/tree-updater.rkt +++ b/lib/tree-updater.rkt @@ -58,6 +58,16 @@ (data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png") (class "thumbimage"))))) (figcaption "Test figure!")) + (div (@ (type "slideshow") (position "center") (widths "500") (mode "slideshow") (seq-no "0") (id "slideshow-0") (hash "b62d0efee427ad7dff1026e6e9dd078c") (class "wikia-slideshow wikia-gallery slideshow-center")) + (div (@ (class "wikia-slideshow-wrapper") (style "width: 500px") (data-test-outer-width)) + (div (@ (class "wikia-slideshow-images-wrapper")) + (ul (@ (class "wikia-slideshow-images neutral") (style "height: 375px; width: 500px") (data-test-inner-width)) + (li (@ (class "wikia-slideshow-first-image")) + (a (@ (class "image lightbox") (title "Expand slideshow") (id "slideshow-0-0") (style "width: 420px")) + (img (@ (data-src "https://static.wikia.nocookie.net/example/images/3/3d/Image.jpg/revision/latest/scale-to-width-down/500?cb=20140129105112") (class "thumbimage") (width "500") (height "281") (style "border: 0px;")))) + (div (@ (class "wikia-slideshow-overlay")) + (div (@ (class "wikia-slideshow-image-caption")) + "Example caption"))))))) (iframe (@ (src "https://example.com/iframe-src"))) (div (@ (class "reviews")) (header "GameSpot Expert Reviews")) diff --git a/src/config.rkt b/src/config.rkt index b1afe0a..0bbdfa3 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -49,6 +49,12 @@ (feature_offline::only . "false") (feature_offline::search . "fandom") + (feature_jsonp::enabled . "true") + + (captcha::enabled . "false") + (captcha::log . "false") + (captcha::ip_header . "") + (access_log::enabled . "false") (promotions::indie_wiki_buddy . "banner home"))) diff --git a/src/data.rkt b/src/data.rkt index 63c7f03..44b5bda 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -20,6 +20,7 @@ (struct-out license^) (struct-out head-data^) (struct-out user-cookies^) + data->siteinfo siteinfo-fetch siteinfo-default license-default @@ -66,12 +67,15 @@ ("formatversion" . "2")))) (cond [(= (easy:response-status-code res) 200) (define data (easy:response-json res)) - (siteinfo^ (jp "/query/general/sitename" data) - (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) - (license^ (jp "/query/rightsinfo/text" data) - (jp "/query/rightsinfo/url" data)))] + (data->siteinfo data)] [else siteinfo-default])])) +(define (data->siteinfo data) + (siteinfo^ (jp "/query/general/sitename" data) + (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) + (license^ (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data)))) + (define/memoize (head-data-getter wikiname) #:hash hash ;; data will be stored here, can be referenced by the memoized closure (define this-data head-data-default) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index a967095..0bbf8c8 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -56,15 +56,25 @@ (sequencer:make subdomain-dispatcher (pathprocedure:make "/" (page ds page-home)) + (filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher)) + (filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image))) + (filter:make (pregexp "^/captcha/verify/[0-9]+/[0-9]+/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-verify))) + (if (config-true? 'captcha::enabled) + (lift:make (page ds page-captcha)) + (λ (_conn _req) (next-dispatcher))) (pathprocedure:make "/proxy" (page ds page-proxy)) (pathprocedure:make "/search" (page ds page-global-search)) (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings)) (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works)) + (pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category))) (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file))) (if (config-true? 'feature_offline::enabled) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) + (if (config-true? 'feature_jsonp::enabled) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp))) + (λ (_conn _req) (next-dispatcher))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home))) diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt new file mode 100644 index 0000000..dcd72be --- /dev/null +++ b/src/page-captcha.rkt @@ -0,0 +1,177 @@ +#lang racket/base +(require racket/class + racket/dict + racket/draw + pict + file/convertible + racket/format + racket/list + racket/math + racket/match + web-server/http + (only-in web-server/dispatchers/dispatch next-dispatcher) + net/url + (only-in net/cookies/server cookie->set-cookie-header cookie-header->alist) + html-writing + "application-globals.rkt" + "data.rkt" + "config.rkt" + "static-data.rkt" + "../lib/url-utils.rkt" + "../lib/xexpr-utils.rkt") + +(provide + page-captcha + page-captcha-image + page-captcha-verify) + +(define (get-ip req) + (define header + (if (config-true? 'captcha::ip_header) + (headers-assq* (string->bytes/utf-8 (config-get 'captcha::ip_header)) (request-headers/raw req)) + #f)) + (if header + (~a (header-value header)) + (request-client-ip req))) + +(define (get-rng req) + (parameterize ([current-pseudo-random-generator (make-pseudo-random-generator)]) + (define ip-segments (regexp-match* "[0-9]+" (get-ip req))) + (define seed + (modulo + (for/sum ([i (in-naturals)] + [s ip-segments]) + (* (add1 i) (add1 (string->number s)))) + (expt 2 32))) + (random-seed seed) + (current-pseudo-random-generator))) + +(define (get-key-solution req) + (parameterize ([current-pseudo-random-generator (get-rng req)]) + (random 1 (add1 9)))) + +(define diameter 35) +(define font (make-object font% 12 'system)) +(define msg "I'm not a robot!") +(define checkbox (filled-ellipse diameter diameter #:color "Pale Goldenrod")) +(define assembly + (frame + (inset + (hc-append + 8 + checkbox + (text msg font)) + 8))) +(define-values (inner-x inner-y) (cc-find assembly checkbox)) +(define-values (lt-x lt-y) (lt-find assembly checkbox)) +(define-values (rb-x rb-y) (rb-find assembly checkbox)) + +(define (get-coordinate-solution req w h) + (parameterize ([current-pseudo-random-generator (get-rng req)]) + (values (random (exact-truncate lt-x) (exact-truncate (- w rb-x))) + (random (exact-truncate lt-y) (exact-truncate (- h rb-y)))))) + +(define (page-captcha req) + (define cookie-header (headers-assq* #"cookie" (request-headers/raw req))) + (define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null)) + (for ([pair cookies-alist]) + (match pair + [(cons "captcha" method) + (when (config-true? 'captcha::log) + (printf "captcha skip - via ~a [~a] - ~v~n" method (get-ip req) (url->string (request-uri req)))) + (next-dispatcher)] + [_ (void)])) + (response-handler + (define body + `(*TOP* + (*DECL* DOCTYPE html) + (html + (head + (meta (@ (name "viewport") (content "width=device-width, initial-scale=1"))) + (title "Checking you're not a bot...") + (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css")))) + (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) + (link (@ (rel "icon") (href ,(head-data^-icon-url head-data-default)))) + (script (@ (defer) (src "/static/captcha.js"))) + (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) + (div (@ (class "main-container")) + (div (@ (class "fandom-community-header__background tileBoth header"))) + (div (@ (class "page")) + (main (@ (class "page__main")) + (div (@ (class "custom-top")) + (h1 (@ (class "page-title")) + "Checking you're not a bot...")) + (div (@ (id "content") #;(class "page-content")) + (div (@ (id "mw-content-text")) + (p "To confirm, please click directly in the circle, or hold down the " ,(~a (get-key-solution req)) " key on your keyboard.") + (noscript (p "JavaScript is required for the captcha. Sorry!")) + (div (@ (id "captcha-area"))))) + ,(application-footer #f))))))))) + (when (config-true? 'debug) + (xexp->html body)) + (response/output + #:code 200 + #:headers always-headers + (λ (out) + (write-html body out))))) + +(define (page-captcha-image req) + (response-handler + (define w (string->number (path/param-path (third (url-path (request-uri req)))))) + (define h (string->number (path/param-path (fourth (url-path (request-uri req)))))) + (define-values (at-x at-y) (get-coordinate-solution req w h)) + (when (config-true? 'captcha::log) + (printf "captcha show - size ~a x ~a - solution ~a x ~a [~a]~n" w h at-x at-y (get-ip req))) + #;(printf "target: ~a x ~a~ncanvas: ~a x ~a~npict size: ~a-~a ~a-~a~n" at-x at-y x y lt-x rb-x lt-y rb-y) + (define dc (make-object bitmap-dc% #f)) + (send dc set-font font) + (define bm (make-object bitmap% w h #f #f)) + (send dc set-bitmap bm) + (draw-pict + assembly + dc + (- at-x inner-x) + (- at-y inner-y)) + (define image (convert bm 'png-bytes)) + (response/output + #:mime-type #"image/png" + #:headers (list (header #"Cache-Control" #"no-cache")) + (λ (out) (display image out))))) + +(define (page-captcha-verify req) + (response-handler + (match-define (list w h x y) (for/list ([segment (drop (url-path (request-uri req)) 2)]) + (string->number (path/param-path segment)))) + #;(printf "solution: ~a x ~a~ncoordinate: ~a x ~a~ndist^2: ~a~n" solution-x solution-y x y dist) + (define headers + (build-headers + always-headers + (cond + [(and (= y 0) (= x (get-key-solution req))) + (when (config-true? 'captcha::log) + (printf "captcha pass - key ~a [~a]~n" x (get-ip req))) + + (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10))))] + [else + (when (config-true? 'captcha::log) + (printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))]) + (when (> y 0) + (let-values ([(solution-x solution-y) (get-coordinate-solution req w h)]) + (let ([dist (+ (expt (- x solution-x) 2) (expt (- y solution-y) 2))]) + (cond + [(dist . < . (expt (/ diameter 2) 2)) + (when (config-true? 'captcha::log) + (printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req))) + (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "coordinate" #:path "/" #:max-age (* 60 60 24 365 10))))] + [else + (when (config-true? 'captcha::log) + (printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req)))])))))) + (match (dict-ref (url-query (request-uri req)) 'from #f) + [(? string? dest) + (response/output + #:code 302 + #:mime-type #"text/plain" + #:headers (cons (header #"Location" (string->bytes/utf-8 dest)) headers) + (λ (out) + (displayln "Checking your answer..." out)))] + [#f (next-dispatcher)]))) diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt new file mode 100644 index 0000000..3e49abf --- /dev/null +++ b/src/page-wiki-jsonp.rkt @@ -0,0 +1,68 @@ +#lang racket/base +(require racket/list + racket/string + web-server/http + net/url-structs + html-writing + "application-globals.rkt" + "data.rkt" + "config.rkt" + "../lib/url-utils.rkt" + "../lib/xexpr-utils.rkt" + "../lib/archive-file-mappings.rkt") + +(provide + page-wiki-jsonp) + +(define (page-wiki-jsonp req) + (response-handler + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) + (define path (string-join (cdr segments) "/")) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) + + (define wiki-page-script-url + (format "https://~a.fandom.com/api.php?~a" + wikiname + (params->query `(("action" . "parse") + ("page" . ,path) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json") + ("callback" . "wikiPageCallback"))))) + (define siteinfo-script-url + (format "https://~a.fandom.com/api.php?~a" + wikiname + (params->query `(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2") + ("callback" . "siteinfoCallback"))))) + + (define body + (generate-wiki-page + `(div + (noscript "You have to enable JavaScript to load wiki pages. Sorry!") + (div (@ (id "loading"))) + (progress (@ (id "progress") (style "margin-bottom: 50vh"))) + (script ,(format #<guess-title segments) + #:siteinfo siteinfo-default)) + (when (config-true? 'debug) + (xexp->html body)) + (response/output + #:code 200 + #:headers always-headers + (λ (out) + (write-html body out))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index da63617..82956e4 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -6,6 +6,7 @@ racket/string ; libs (prefix-in easy: net/http-easy) + json ; html libs "../lib/html-parsing/main.rkt" html-writing @@ -19,8 +20,6 @@ "data.rkt" "fandom-request.rkt" "../lib/archive-file-mappings.rkt" - "../lib/pure-utils.rkt" - "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/tree-updater.rkt" "../lib/url-utils.rkt" @@ -29,6 +28,7 @@ (provide ; used by the web server page-wiki + page-wiki-with-data ; used by page-category, and similar pages that are partially wiki pages update-tree-wiki preprocess-html-wiki) @@ -59,48 +59,10 @@ (cond [(eq? 200 (easy:response-status-code dest-res)) - (let* ([data (easy:response-json dest-res)] - [title (jp "/parse/title" data "")] - [page-html (jp "/parse/text" data "")] - [page-html (preprocess-html-wiki page-html)] - [page (html->xexp page-html)] - [head-data ((head-data-getter wikiname) data)]) + (let ([data (easy:response-json dest-res)]) (if (equal? "missingtitle" (jp "/error/code" data #f)) (next-dispatcher) - (response-handler - (define body - (generate-wiki-page - (update-tree-wiki page wikiname) - #:req req - #:source-url source-url - #:wikiname wikiname - #:title title - #:head-data head-data - #:siteinfo siteinfo)) - (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) - (define redirect-msg-a (if redirect-msg - ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) - #f)) - (define headers - (build-headers - always-headers - ; redirect-query-parameter: only the string "no" is significant: - ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg-a - (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] - [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (header #"Refresh" value))))) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - #:headers headers - (λ (out) - (write-html body out))))))] + (take-json-rewrite-and-return-page data)))] [(eq? 404 (easy:response-status-code dest-res)) (next-dispatcher)] [(memq (easy:response-status-code dest-res) '(403 406)) @@ -132,3 +94,55 @@ path (easy:response-status-code dest-res) (easy:response-body dest-res)))])) + +(define (page-wiki-with-data req) + (define post-data/bytes (request-post-data/raw req)) + (define post-data/string (bytes->string/utf-8 post-data/bytes)) + (define post-data (string->jsexpr post-data/string)) + (define wikiname (jp "/wikiname" post-data)) + (define path (jp "/path" post-data)) + (take-json-rewrite-and-return-page + #:req req + #:wikiname wikiname + #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) + #:data (jp "/data" post-data) + #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))) + +(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo) + (define title (jp "/parse/title" data "")) + (define page-html (preprocess-html-wiki (jp "/parse/text" data ""))) + (define page (html->xexp page-html)) + (define head-data ((head-data-getter wikiname) data)) + (response-handler + (define body + (generate-wiki-page + (update-tree-wiki page wikiname) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:head-data head-data + #:siteinfo siteinfo)) + (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) + (define html (xexp->html-bytes body)) + (define headers + (build-headers + always-headers + ; redirect-query-parameter: only the string "no" is significant: + ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 + (when (and redirect-msg-a + (not (equal? redirect-query-parameter "no"))) + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] + [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) + (header #"Refresh" value))))) + (response/full + 200 + #"OK" + (current-seconds) + #"text/html; charset=utf-8" + headers + (list html)))) diff --git a/static/captcha.js b/static/captcha.js new file mode 100644 index 0000000..87f6b88 --- /dev/null +++ b/static/captcha.js @@ -0,0 +1,24 @@ +const u = new URL(location) +const from = u.searchParams.get("from") || location.href +let answered = false + +const area = document.getElementById("captcha-area") +const areaBox = area.getBoundingClientRect() +const width = Math.floor(areaBox.width) +const height = Math.floor(window.innerHeight - areaBox.bottom - areaBox.left) +const img = document.createElement("img") +img.src = `/captcha/img/${width}/${height}` +img.addEventListener("click", event => { + if (answered) return + answered = true + location = `/captcha/verify/${width}/${height}/${event.offsetX}/${event.offsetY}?` + new URLSearchParams({from}) +}) +area.appendChild(img) + +document.addEventListener("keydown", event => { + if (event.repeat) { + if (answered) return + answered = true + location = `/captcha/verify/0/0/${event.key}/0?` + new URLSearchParams({from}) + } +}) diff --git a/static/jsonp.js b/static/jsonp.js new file mode 100644 index 0000000..36140d9 --- /dev/null +++ b/static/jsonp.js @@ -0,0 +1,57 @@ +const loading = document.getElementById("loading") +loading.textContent = "Loading, please wait..." +const progress = document.getElementById("progress") + +let wikiPage = null +function wikiPageCallback(data) { + wikiPage = data + cont() +} + +let siteinfo = null +function siteinfoCallback(data) { + siteinfo = data + cont() +} + +async function cont() { + if (!(wikiPage && siteinfo)) return + + const xhr = new XMLHttpRequest(); + + const uploadFraction = 0.7 + + // Upload progress + xhr.upload.addEventListener("progress", event => { + if (event.lengthComputable) { + progress.value = (event.loaded / event.total) * uploadFraction + console.log( + `Uploaded ${((event.loaded / event.total) * 100).toFixed(2)}%`, + ) + } + }) + + // Download progress + xhr.addEventListener("progress", event => { + if (event.lengthComputable) { + progress.value = (event.loaded / event.total) * (1 - uploadFraction) + uploadFraction + console.log( + `Downloaded ${((event.loaded / event.total) * 100).toFixed(2)}%`, + ) + } + }) + + xhr.addEventListener("load", () => { + console.log(xhr) + document.body = xhr.responseXML.body + }) + + xhr.open("POST", "/api/render/wiki") + xhr.responseType = "document" + xhr.send(JSON.stringify({ + data: wikiPage, + siteinfo, + wikiname, + path + })); +} diff --git a/static/robots.txt b/static/robots.txt index b0e8bf6..b62f622 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -2,3 +2,4 @@ User-Agent: * Disallow: /*/wiki/* Disallow: /proxy Disallow: /set-user-settings +Disallow: /captcha From 1c675d487361a3903eda15eac6cb3c77a8631cd1 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Nov 2025 23:12:30 +1300 Subject: [PATCH 03/30] update dist --- dist.rkt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dist.rkt b/dist.rkt index 2e46f8c..491ca80 100644 --- a/dist.rkt +++ b/dist.rkt @@ -3,6 +3,7 @@ "src/config.rkt" "src/dispatcher-tree.rkt") +(require (only-in "src/page-captcha.rkt" page-captcha page-captcha-image page-captcha-verify)) (require (only-in "src/page-category.rkt" page-category)) (require (only-in "src/page-global-search.rkt" page-global-search)) (require (only-in "src/page-home.rkt" page-home)) @@ -15,8 +16,9 @@ (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-static-archive.rkt" page-static-archive)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) -(require (only-in "src/page-wiki.rkt" page-wiki)) +(require (only-in "src/page-wiki.rkt" page-wiki page-wiki-with-data)) (require (only-in "src/page-wiki-offline.rkt" page-wiki-offline)) +(require (only-in "src/page-wiki-jsonp.rkt" page-wiki-jsonp)) (require (only-in "src/page-file.rkt" page-file)) (serve/launch/wait @@ -27,6 +29,9 @@ (λ (quit) (dispatcher-tree ; order of these does not matter + page-captcha + page-captcha-image + page-captcha-verify page-category page-global-search page-home @@ -38,6 +43,8 @@ page-static-archive page-wiki page-wiki-offline + page-wiki-with-data + page-wiki-jsonp page-file redirect-wiki-home static-dispatcher From 143fadcafba22424794cbb846380180a6046224e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Nov 2025 23:16:15 +1300 Subject: [PATCH 04/30] clear bogus log --- src/page-captcha.rkt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt index dcd72be..91889b6 100644 --- a/src/page-captcha.rkt +++ b/src/page-captcha.rkt @@ -152,9 +152,10 @@ (printf "captcha pass - key ~a [~a]~n" x (get-ip req))) (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10))))] - [else + [(= y 0) (when (config-true? 'captcha::log) - (printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))]) + (printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))] + [else (void)]) (when (> y 0) (let-values ([(solution-x solution-y) (get-coordinate-solution req w h)]) (let ([dist (+ (expt (- x solution-x) 2) (expt (- y solution-y) 2))]) From 4f4c939631fa17b1bf140a76db712ba61994bb11 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Nov 2025 23:35:58 +1300 Subject: [PATCH 05/30] also update document head --- static/jsonp.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/static/jsonp.js b/static/jsonp.js index 36140d9..3dde75b 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -25,9 +25,6 @@ async function cont() { xhr.upload.addEventListener("progress", event => { if (event.lengthComputable) { progress.value = (event.loaded / event.total) * uploadFraction - console.log( - `Uploaded ${((event.loaded / event.total) * 100).toFixed(2)}%`, - ) } }) @@ -35,15 +32,19 @@ async function cont() { xhr.addEventListener("progress", event => { if (event.lengthComputable) { progress.value = (event.loaded / event.total) * (1 - uploadFraction) + uploadFraction - console.log( - `Downloaded ${((event.loaded / event.total) * 100).toFixed(2)}%`, - ) } }) xhr.addEventListener("load", () => { console.log(xhr) document.body = xhr.responseXML.body + document.title = xhr.responseXML.title + for (const e of xhr.responseXML.head.children) { + if (["LINK"].includes(e.tagName)) { + const imported = document.importNode(e, true) + document.head.appendChild(imported) + } + } }) xhr.open("POST", "/api/render/wiki") From 02848acfbb6c7da913dba6eb7cc2c87c36dca5c5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 4 Nov 2025 23:56:32 +1300 Subject: [PATCH 06/30] make sure it's a post --- src/page-wiki.rkt | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 82956e4..707157a 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -96,17 +96,20 @@ (easy:response-body dest-res)))])) (define (page-wiki-with-data req) - (define post-data/bytes (request-post-data/raw req)) - (define post-data/string (bytes->string/utf-8 post-data/bytes)) - (define post-data (string->jsexpr post-data/string)) - (define wikiname (jp "/wikiname" post-data)) - (define path (jp "/path" post-data)) - (take-json-rewrite-and-return-page - #:req req - #:wikiname wikiname - #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) - #:data (jp "/data" post-data) - #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))) + (response-handler + (define post-data/bytes (request-post-data/raw req)) + (when (not post-data/bytes) + (raise-user-error 'page-wiki-with-data "POST requests only, please.")) + (define post-data/string (bytes->string/utf-8 post-data/bytes)) + (define post-data (string->jsexpr post-data/string)) + (define wikiname (jp "/wikiname" post-data)) + (define path (jp "/path" post-data)) + (take-json-rewrite-and-return-page + #:req req + #:wikiname wikiname + #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) + #:data (jp "/data" post-data) + #:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))) (define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo) (define title (jp "/parse/title" data "")) From c4e2fb00ef5fe7d01efe00e2a2875e696d00584d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 5 Nov 2025 00:04:19 +1300 Subject: [PATCH 07/30] only replace content so suggestions work --- static/jsonp.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/jsonp.js b/static/jsonp.js index 3dde75b..4ed99de 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -37,7 +37,8 @@ async function cont() { xhr.addEventListener("load", () => { console.log(xhr) - document.body = xhr.responseXML.body + const imported = document.importNode(xhr.responseXML.getElementById("content"), true) + document.getElementById("content").replaceWith(imported) document.title = xhr.responseXML.title for (const e of xhr.responseXML.head.children) { if (["LINK"].includes(e.tagName)) { From 048709b2d1cbda604585f83bbc82d7cd5c510baf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 5 Nov 2025 00:06:49 +1300 Subject: [PATCH 08/30] fix cache control for jsonp.js --- src/page-wiki-jsonp.rkt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt index 3e49abf..7ff425b 100644 --- a/src/page-wiki-jsonp.rkt +++ b/src/page-wiki-jsonp.rkt @@ -9,7 +9,8 @@ "config.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt" - "../lib/archive-file-mappings.rkt") + "../lib/archive-file-mappings.rkt" + "static-data.rkt") (provide page-wiki-jsonp) @@ -51,7 +52,7 @@ var wikiname = ~v; var path = ~v; END wikiname path)) - (script (@ (src "/static/jsonp.js"))) + (script (@ (src ,(get-static-url "jsonp.js")))) (script (@ (async) (src ,wiki-page-script-url))) (script (@ (async) (src ,siteinfo-script-url)))) #:req req From 1dd90f5a7d0ae62b78f4327997a79ab0a6986540 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 5 Nov 2025 16:35:17 +1300 Subject: [PATCH 09/30] Refactor jsonp js for cuteness --- src/application-globals.rkt | 7 +++- src/page-captcha.rkt | 2 +- src/page-wiki-jsonp.rkt | 22 ++++++----- src/page-wiki.rkt | 44 +++++++++++++++------- static/jsonp.js | 75 ++++++++++++++++++++++++++++--------- 5 files changed, 106 insertions(+), 44 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 26cd6ae..5884804 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -163,11 +163,13 @@ #:head-data [head-data-in #f] #:siteinfo [siteinfo-in #f] #:user-cookies [user-cookies-in #f] - #:online-styles [online-styles #t]) + #:online-styles [online-styles #t] + #:path [path-in #f]) (define siteinfo (or siteinfo-in siteinfo-default)) (define head-data (or head-data-in ((head-data-getter wikiname)))) (define user-cookies (or user-cookies-in (user-cookies-getter req))) (define origin (format "https://~a.fandom.com" wikiname)) + (define path (or path-in "")) (define required-styles (cond [online-styles @@ -197,7 +199,8 @@ (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) (script "const BWData = " ,(jsexpr->string (hasheq 'wikiname wikiname - 'strict_proxy (config-true? 'strict_proxy)))) + 'strict_proxy (config-true? 'strict_proxy) + 'path path))) ,(if (config-true? 'feature_search_suggestions) `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js")))) "") diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt index 91889b6..3c8f2f5 100644 --- a/src/page-captcha.rkt +++ b/src/page-captcha.rkt @@ -92,7 +92,7 @@ (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css")))) (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) (link (@ (rel "icon") (href ,(head-data^-icon-url head-data-default)))) - (script (@ (defer) (src "/static/captcha.js"))) + (script (@ (defer) (src ,(get-static-url "captcha.js")))) (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) (div (@ (class "main-container")) (div (@ (class "fandom-community-header__background tileBoth header"))) diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt index 7ff425b..0c7918c 100644 --- a/src/page-wiki-jsonp.rkt +++ b/src/page-wiki-jsonp.rkt @@ -30,7 +30,7 @@ ("prop" . "text|headhtml|langlinks") ("formatversion" . "2") ("format" . "json") - ("callback" . "wikiPageCallback"))))) + ("callback" . "proxy.wikipage"))))) (define siteinfo-script-url (format "https://~a.fandom.com/api.php?~a" wikiname @@ -39,27 +39,29 @@ ("siprop" . "general|rightsinfo") ("format" . "json") ("formatversion" . "2") - ("callback" . "siteinfoCallback"))))) + ("callback" . "proxy.siteinfo"))))) (define body (generate-wiki-page `(div (noscript "You have to enable JavaScript to load wiki pages. Sorry!") (div (@ (id "loading"))) - (progress (@ (id "progress") (style "margin-bottom: 50vh"))) - (script ,(format #< obj[prop] = value }}) END - wikiname path)) - (script (@ (src ,(get-static-url "jsonp.js")))) + ) (script (@ (async) (src ,wiki-page-script-url))) - (script (@ (async) (src ,siteinfo-script-url)))) + (script (@ (async) (src ,siteinfo-script-url))) + (script (@ (type "module") (src ,(get-static-url "jsonp.js"))))) #:req req #:source-url source-url #:wikiname wikiname #:title (url-segments->guess-title segments) - #:siteinfo siteinfo-default)) + #:siteinfo siteinfo-default + #:path path)) (when (config-true? 'debug) (xexp->html body)) (response/output diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 707157a..1caf41d 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -97,19 +97,37 @@ (define (page-wiki-with-data req) (response-handler - (define post-data/bytes (request-post-data/raw req)) - (when (not post-data/bytes) - (raise-user-error 'page-wiki-with-data "POST requests only, please.")) - (define post-data/string (bytes->string/utf-8 post-data/bytes)) - (define post-data (string->jsexpr post-data/string)) - (define wikiname (jp "/wikiname" post-data)) - (define path (jp "/path" post-data)) - (take-json-rewrite-and-return-page - #:req req - #:wikiname wikiname - #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) - #:data (jp "/data" post-data) - #:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))) + (let/cc return + (define post-data/bytes (request-post-data/raw req)) + (when (not post-data/bytes) + (return (response/jsexpr + #:code 400 + #:headers always-headers + '#hasheq((error . + #hasheq((code . "breezewiki") + (info . "POST requests only, please."))))))) + + (define origin-header + (or (headers-assq* #"origin" (request-headers/raw req)) + (headers-assq* #"referer" (request-headers/raw req)))) + (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin)))) + (return (response/jsexpr + #:code 400 + #:headers always-headers + '#hasheq((error . + #hasheq((code . "breezewiki") + (info . "Origin/Referer header failed validation - cross-origin requests are not allowed here"))))))) + + (define post-data/string (bytes->string/utf-8 post-data/bytes)) + (define post-data (string->jsexpr post-data/string)) + (define wikiname (jp "/wikiname" post-data)) + (define path (jp "/path" post-data)) + (take-json-rewrite-and-return-page + #:req req + #:wikiname wikiname + #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) + #:data (jp "/data" post-data) + #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))))) (define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo) (define title (jp "/parse/title" data "")) diff --git a/static/jsonp.js b/static/jsonp.js index 4ed99de..851f86e 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -1,21 +1,40 @@ -const loading = document.getElementById("loading") -loading.textContent = "Loading, please wait..." -const progress = document.getElementById("progress") +import {h, htm, render, signal, computed, effect} from "./preact.js" +const html = htm.bind(h) -let wikiPage = null -function wikiPageCallback(data) { - wikiPage = data - cont() -} +// *** Loading indicator -let siteinfo = null -function siteinfoCallback(data) { - siteinfo = data - cont() -} +render(html`Loading, please wait...`, document.getElementById("loading")) + +// *** Progress bar + +const progress = signal(null) + +const progressBar = document.getElementById("progress-bar") +while (progressBar.childNodes[0] !== undefined) progressBar.childNodes[0].remove() // clear out loading indicators + +render(html``, progressBar) + +// *** Incoming data processing + +// Handle case where data is immediately available +cont() + +// Handle case where data may become available in the future +window.proxy = new Proxy(jsonpData, { + get(obj, prop) { + return value => { + obj[prop] = value + cont() + } + } +}) + +// *** Data upload and download async function cont() { - if (!(wikiPage && siteinfo)) return + if (jsonpData.wikipage?.error) return error(jsonpData.wikipage) + if (jsonpData.siteinfo?.error) return error(jsonpData.siteinfo) + if (!(jsonpData.wikipage && jsonpData.siteinfo)) return const xhr = new XMLHttpRequest(); @@ -46,14 +65,34 @@ async function cont() { document.head.appendChild(imported) } } + const redirectTo = document.querySelector("#content .redirectMsg a") + if (redirectTo) { + redirectTo.click() + } }) xhr.open("POST", "/api/render/wiki") xhr.responseType = "document" xhr.send(JSON.stringify({ - data: wikiPage, - siteinfo, - wikiname, - path + data: jsonpData.wikipage, + siteinfo: jsonpData.siteinfo, + wikiname: BWData.wikiname, + path: BWData.path })); } + +function error(data) { + const eContent = document.getElementById("content") + while (eContent.childNodes[0] !== undefined) eContent.childNodes[0].remove() // clear out loading indicators + document.title = `Error | BreezeWiki` + render(html` +${data.error.code === "missingtitle" +? html`

This page doesn't exist on Fandom.

` +: html` +

BreezeWiki wasn't able to load this page.

+

${data.error.code}: ${data.error.info}

+` +} +

Return to the homepage?

+`, eContent) +} From 5c3ff9b37cf7e99773c0e3d9d7ea0e735a215a87 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 8 Nov 2025 13:43:12 +1300 Subject: [PATCH 10/30] Fix tabs for jsonp --- src/application-globals.rkt | 9 ++++++--- src/page-wiki-jsonp.rkt | 3 ++- src/static-data.rkt | 23 ++++++++++++++++----- static/countdown.js | 2 +- static/jsonp.js | 8 +++++++- static/search-suggestions.js | 2 +- static/tabs.js | 39 +++++++++++++++++++++++++----------- 7 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 5884804..0d6b3dd 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -164,7 +164,8 @@ #:siteinfo [siteinfo-in #f] #:user-cookies [user-cookies-in #f] #:online-styles [online-styles #t] - #:path [path-in #f]) + #:path [path-in #f] + #:jsonp [jsonp-in #f]) (define siteinfo (or siteinfo-in siteinfo-default)) (define head-data (or head-data-in ((head-data-getter wikiname)))) (define user-cookies (or user-cookies-in (user-cookies-getter req))) @@ -200,12 +201,14 @@ (script "const BWData = " ,(jsexpr->string (hasheq 'wikiname wikiname 'strict_proxy (config-true? 'strict_proxy) - 'path path))) + 'path path + 'jsonp jsonp-in))) + (script (@ (type "importmap")) ,importmap) ,(if (config-true? 'feature_search_suggestions) `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js")))) "") (script (@ (type "module") (src ,(get-static-url "countdown.js")))) - (script (@ (defer) (src ,(get-static-url "tabs.js")))) + (script (@ (type "module") (src ,(get-static-url "tabs.js")))) (link (@ (rel "icon") (href ,(u (λ (v) (config-true? 'strict_proxy)) (λ (v) (u-proxy-url v)) (head-data^-icon-url head-data)))))) diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt index 0c7918c..ada846e 100644 --- a/src/page-wiki-jsonp.rkt +++ b/src/page-wiki-jsonp.rkt @@ -61,7 +61,8 @@ END #:wikiname wikiname #:title (url-segments->guess-title segments) #:siteinfo siteinfo-default - #:path path)) + #:path path + #:jsonp #t)) (when (config-true? 'debug) (xexp->html body)) (response/output diff --git a/src/static-data.rkt b/src/static-data.rkt index 1255ca5..ec7eede 100644 --- a/src/static-data.rkt +++ b/src/static-data.rkt @@ -1,16 +1,20 @@ #lang typed/racket/base (require racket/path racket/runtime-path - racket/string) + racket/string + typed/json) (provide get-static-url + importmap link-header) (define-runtime-path path-static "../static") (define static-data - (for/hash : (Immutable-HashTable Path Nonnegative-Integer)([f (directory-list path-static)]) + (for/hash : (Immutable-HashTable Path Nonnegative-Integer) + ([f (directory-list path-static)] + #:when (not (regexp-match? #rx"^.#|^#|~$" (path->string f)))) (define built (simple-form-path (build-path path-static f))) (values built (file-or-directory-modify-seconds built)))) @@ -21,12 +25,21 @@ (build-path path-static path-or-filename)))) (format "/static/~a?t=~a" (file-name-from-path the-path) (hash-ref static-data the-path))) +(: importmap String) +(define importmap + (jsexpr->string + `#hasheq((imports . ,(for/hasheq : (Immutable-HashTable Symbol String) + ([(k v) (in-hash static-data)] + #:when (equal? (path-get-extension k) #".js")) + (values (string->symbol (path->string (path-replace-extension (assert (file-name-from-path k) path?) #""))) + (get-static-url k))))))) + ; https://developer.mozilla.org/en-US/docs/Web/HTML/Link_types/preload (: link-header String) (define link-header - (let* ([with-t '(("main.css" "as=style"))] - [without-t '(("preact.js" "as=script") - ("source-sans-pro-v21-vietnamese_latin-ext_latin_greek-ext_greek_cyrillic-ext_cyrillic-regular.woff2" "as=font" "crossorigin" "type=font/woff2"))] + (let* ([with-t '(("main.css" "as=style") + ("preact.js" "as=script"))] + [without-t '(("source-sans-pro-v21-vietnamese_latin-ext_latin_greek-ext_greek_cyrillic-ext_cyrillic-regular.woff2" "as=font" "crossorigin" "type=font/woff2"))] [with-t-full (map (λ ([path : (Listof String)]) (cons (get-static-url (car path)) (cdr path))) with-t)] [without-t-full (map (λ ([path : (Listof String)]) (cons (format "/static/~a" (car path)) (cdr path))) without-t)] [all (append with-t-full without-t-full)] diff --git a/static/countdown.js b/static/countdown.js index ad8d0be..92b9c06 100644 --- a/static/countdown.js +++ b/static/countdown.js @@ -2,7 +2,7 @@ // sample: bandori/wiki/BanG_Dream!_Wikia // sample: ensemble-stars/wiki/The_English_Ensemble_Stars_Wiki -import {h, htm, render, signal, computed, effect} from "./preact.js" +import {h, htm, render, signal, computed, effect} from "preact" const html = htm.bind(h) const now = signal(Date.now()) diff --git a/static/jsonp.js b/static/jsonp.js index 851f86e..a183b0f 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -1,6 +1,11 @@ -import {h, htm, render, signal, computed, effect} from "./preact.js" +import {h, htm, render, signal, computed, effect} from "preact" const html = htm.bind(h) +// *** Status + +const loaded = signal(false) +export {loaded} + // *** Loading indicator render(html`Loading, please wait...`, document.getElementById("loading")) @@ -69,6 +74,7 @@ async function cont() { if (redirectTo) { redirectTo.click() } + loaded.value = true }) xhr.open("POST", "/api/render/wiki") diff --git a/static/search-suggestions.js b/static/search-suggestions.js index e0a32d6..ddd2927 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -1,4 +1,4 @@ -import {h, htm, render, signal, computed, effect} from "./preact.js" +import {h, htm, render, signal, computed, effect} from "preact" const html = htm.bind(h) const classNames = classArr => classArr.filter(el => el).join(" ") diff --git a/static/tabs.js b/static/tabs.js index 718b48e..bbf5d2a 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -1,19 +1,22 @@ -"use strict"; +import {effect} from "preact" +import {loaded} from "jsonp" const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null -for (const tabber of document.body.querySelectorAll(".wds-tabber")) { - for (const [tab, content] of getTabberTabs(tabber)) { - // set up click listener on every tab - tab.addEventListener("click", e => { - setCurrentTab(tabber, tab, content) - e.preventDefault() - }) +function setUpAllTabs() { + for (const tabber of document.body.querySelectorAll(".wds-tabber")) { + for (const [tab, content] of getTabberTabs(tabber)) { + // set up click listener on every tab + tab.addEventListener("click", e => { + setCurrentTab(tabber, tab, content) + e.preventDefault() + }) - // re-open a specific tab on page load based on the URL hash - if (tab.dataset.hash === tabFromHash) { - setCurrentTab(tabber, tab, content) - tab.scrollIntoView() + // re-open a specific tab on page load based on the URL hash + if (tab.dataset.hash === tabFromHash) { + setCurrentTab(tabber, tab, content) + tab.scrollIntoView() + } } } } @@ -37,4 +40,16 @@ function setCurrentTab(tabber, tab, content) { } } +if (!BWData.jsonp) { + setUpAllTabs() +} else if (loaded.value) { + setUpAllTabs() +} else { + effect(() => { + if (loaded.value) { + setUpAllTabs() + } + }) +} + document.body.classList.remove("bw-tabs-nojs") From c93c901ad309838c28fe01543e0458b3c9231c0d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 8 Nov 2025 13:55:55 +1300 Subject: [PATCH 11/30] Load first suggestion when pressing enter --- static/search-suggestions.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/static/search-suggestions.js b/static/search-suggestions.js index ddd2927..ae8be36 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -1,4 +1,4 @@ -import {h, htm, render, signal, computed, effect} from "preact" +import {h, htm, render, signal, computed, effect, useSignalEffect} from "preact" const html = htm.bind(h) const classNames = classArr => classArr.filter(el => el).join(" ") @@ -13,6 +13,7 @@ const query = signal("") const focus = signal(false) const st = signal("ready") const suggestions = signal([]) +const enterWasLastKey = signal(false) // processing functions @@ -49,6 +50,12 @@ function acceptSuggestion(hit) { // suggestion list view function Suggestion(hit) { + useSignalEffect(() => { + if (enterWasLastKey.value && st.value === "ready") { + enterWasLastKey.value.preventDefault() + acceptSuggestion(hit) + } + }) return html`
    • ` } @@ -79,9 +86,15 @@ window.addEventListener("pageshow", () => { st.value = "ready" // unlock results from changing after returning to page }) +effect(() => { + if (enterWasLastKey.value && st.value === "loading") { + enterWasLastKey.value.preventDefault() // wait for results before going + } +}) + function SuggestionInput() { return html` - query.value = e.target.value} value=${query.value} class=${classNames(["bw-ss__input", `bw-ss__input--${st.value}`])} />` + query.value = e.target.value} onKeyDown=${e => enterWasLastKey.value = e.key === "Enter" && e} value=${query.value} class=${classNames(["bw-ss__input", `bw-ss__input--${st.value}`])} />` } render(html`<${SuggestionInput} />`, eInput) From 0524e60d5d74359d63f8b9413b6c10df96899670 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 19:09:24 +1300 Subject: [PATCH 12/30] Update appearance of external links --- lib/tree-updater.rkt | 10 ++++++++++ static/jsonp.js | 3 ++- static/main.css | 8 ++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/tree-updater.rkt b/lib/tree-updater.rkt index 2abf56c..f941285 100644 --- a/lib/tree-updater.rkt +++ b/lib/tree-updater.rkt @@ -235,6 +235,16 @@ (eq? element-type 'use)) element)))) return-no-element] + ; svg icon for external links + [(and (eq? element-type 'a) + (dict-has-key? attributes 'href) + (not (string-contains? (car (dict-ref attributes 'href)) "wikipedia.org")) + (or (has-class? "extiw" attributes) (has-class? "external" attributes))) + `(,element-type + ,attributes + (,@children + (svg (@ (fill "currentColor") (width "12") (height "12") (viewBox "0 0 12 12") (class "external")) + (path (@ (d "M6 1h5v5L8.86 3.85 4.7 8 4 7.3l4.15-4.16zM2 3h2v1H2v6h6V8h1v2a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V4a1 1 0 0 1 1-1"))))))] ; exclude infobox items that are videos, and gallery items that are videos [(and (or (has-class? "pi-item" attributes) (has-class? "wikia-gallery-item" attributes)) diff --git a/static/jsonp.js b/static/jsonp.js index a183b0f..63f4141 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -65,7 +65,8 @@ async function cont() { document.getElementById("content").replaceWith(imported) document.title = xhr.responseXML.title for (const e of xhr.responseXML.head.children) { - if (["LINK"].includes(e.tagName)) { + const alreadyImported = [...document.querySelectorAll("link[href]")].map(e => e.href) + if (e.tagName === "LINK" && !alreadyImported.includes(e.href)) { const imported = document.importNode(e, true) document.head.appendChild(imported) } diff --git a/static/main.css b/static/main.css index 5b3e7e2..267155f 100644 --- a/static/main.css +++ b/static/main.css @@ -174,13 +174,13 @@ img { } /* indicate wikipedia links */ -.extiw::after, .external::after { +svg.external { vertical-align: super; - content: "[🡕]"; - font-family: serif; - font-size: smaller; } .extiw[href*="wikipedia.org"]::after { + vertical-align: super; + font-family: serif; + font-size: smaller; content: "[W]"; } From 7ba59b5d71c66c83bf89a945a18b9d2a06c6f131 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 19:45:20 +1300 Subject: [PATCH 13/30] Better theme, better errors --- static/jsonp.js | 57 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/static/jsonp.js b/static/jsonp.js index 63f4141..99c4885 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -34,6 +34,20 @@ window.proxy = new Proxy(jsonpData, { } }) +// *** Page theme + +// Try to make it a bit more accurate. Normally this is done server-side by sending a `theme=x` cookie with the parse request. But that's not possible in jsonp mode, and there's no equivalent URL query string to set the theme. +// Helps on Minecraft wiki. Might not be complete on some wikis. +cookieStore.get("theme").then(cookie => { + if (cookie && cookie.value !== "default") { + document.body.classList.remove("theme-fandomdesktop-light") + document.body.classList.remove("theme-fandomdesktop-dark") + document.body.removeAttribute("data-theme") + document.body.classList.add(`theme-fandomdesktop-${cookie.value}`) + document.body.setAttribute("data-theme", cookie.value) + } +}) + // *** Data upload and download async function cont() { @@ -45,6 +59,19 @@ async function cont() { const uploadFraction = 0.7 + const pkg = { + url: "/api/render/wiki", + init: { + method: "POST", + body: JSON.stringify({ + data: jsonpData.wikipage, + siteinfo: jsonpData.siteinfo, + wikiname: BWData.wikiname, + path: BWData.path + }) + } + } + // Upload progress xhr.upload.addEventListener("progress", event => { if (event.lengthComputable) { @@ -61,8 +88,11 @@ async function cont() { xhr.addEventListener("load", () => { console.log(xhr) + if (xhr.status === 500) return fetch(pkg.url, pkg.init).then(res => res.text()).then(error) + // page -> #content const imported = document.importNode(xhr.responseXML.getElementById("content"), true) document.getElementById("content").replaceWith(imported) + // document.title = xhr.responseXML.title for (const e of xhr.responseXML.head.children) { const alreadyImported = [...document.querySelectorAll("link[href]")].map(e => e.href) @@ -71,6 +101,7 @@ async function cont() { document.head.appendChild(imported) } } + // Redirects const redirectTo = document.querySelector("#content .redirectMsg a") if (redirectTo) { redirectTo.click() @@ -78,28 +109,20 @@ async function cont() { loaded.value = true }) - xhr.open("POST", "/api/render/wiki") + xhr.open(pkg.init.method, pkg.url) xhr.responseType = "document" - xhr.send(JSON.stringify({ - data: jsonpData.wikipage, - siteinfo: jsonpData.siteinfo, - wikiname: BWData.wikiname, - path: BWData.path - })); + xhr.send(pkg.init.body); } function error(data) { const eContent = document.getElementById("content") while (eContent.childNodes[0] !== undefined) eContent.childNodes[0].remove() // clear out loading indicators document.title = `Error | BreezeWiki` - render(html` -${data.error.code === "missingtitle" -? html`

    This page doesn't exist on Fandom.

    ` -: html` -

    BreezeWiki wasn't able to load this page.

    -

    ${data.error.code}: ${data.error.info}

    -` -} -

    Return to the homepage?

    -`, eContent) + if (typeof data === "string") { + render(html`

    BreezeWiki ran into an error on this page.

    Try reloading the page.

    If this keeps happening, you could send a public bug report. Please include the following information:

    URL: ${window.location.href}${"\n"}${data}
    `, eContent) + } else if (data.error.code === "missingtitle") { + render(html`

    This page doesn't exist on Fandom.

    Return to the homepage?

    `, eContent) + } else { + render(html`

    BreezeWiki wasn't able to load this page.

    ${data.error.code}: ${data.error.info}

    `, eContent) + } } From de6233ad650294920bef471ed965e664b74b047f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Nov 2025 19:55:51 +1300 Subject: [PATCH 14/30] debounce enter in search (mobile) --- static/search-suggestions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/search-suggestions.js b/static/search-suggestions.js index ae8be36..fcd5ee6 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -87,7 +87,7 @@ window.addEventListener("pageshow", () => { }) effect(() => { - if (enterWasLastKey.value && st.value === "loading") { + if (enterWasLastKey.value && (st.value === "loading" || st.value === "accepted")) { enterWasLastKey.value.preventDefault() // wait for results before going } }) From b2fe4ec61a3a962833413ad24684fa1eb4ad1829 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 11 Nov 2025 13:47:53 +1300 Subject: [PATCH 15/30] fix annotations on wikipedia links --- static/main.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/main.css b/static/main.css index 267155f..e89008e 100644 --- a/static/main.css +++ b/static/main.css @@ -177,7 +177,7 @@ img { svg.external { vertical-align: super; } -.extiw[href*="wikipedia.org"]::after { +.external[href*="wikipedia.org"]::after, .extiw[href*="wikipedia.org"]::after { vertical-align: super; font-family: serif; font-size: smaller; From f4907a7b7749aebc9e8122932a681315d367ed22 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 01:06:32 +1300 Subject: [PATCH 16/30] show captcha correctly if page unloads --- src/page-captcha.rkt | 30 +++++++++++++++--------------- static/jsonp.js | 3 +++ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt index 3c8f2f5..df4f5bb 100644 --- a/src/page-captcha.rkt +++ b/src/page-captcha.rkt @@ -92,21 +92,21 @@ (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css")))) (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css")))) (link (@ (rel "icon") (href ,(head-data^-icon-url head-data-default)))) - (script (@ (defer) (src ,(get-static-url "captcha.js")))) - (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) - (div (@ (class "main-container")) - (div (@ (class "fandom-community-header__background tileBoth header"))) - (div (@ (class "page")) - (main (@ (class "page__main")) - (div (@ (class "custom-top")) - (h1 (@ (class "page-title")) - "Checking you're not a bot...")) - (div (@ (id "content") #;(class "page-content")) - (div (@ (id "mw-content-text")) - (p "To confirm, please click directly in the circle, or hold down the " ,(~a (get-key-solution req)) " key on your keyboard.") - (noscript (p "JavaScript is required for the captcha. Sorry!")) - (div (@ (id "captcha-area"))))) - ,(application-footer #f))))))))) + (script (@ (defer) (src ,(get-static-url "captcha.js"))))) + (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal")) + (div (@ (class "main-container")) + (div (@ (class "fandom-community-header__background tileBoth header"))) + (div (@ (class "page")) + (main (@ (class "page__main")) + (div (@ (class "custom-top")) + (h1 (@ (class "page-title")) + "Checking you're not a bot...")) + (div (@ (id "content") #;(class "page-content")) + (div (@ (id "mw-content-text")) + (p "To confirm, please click directly in the circle, or hold down the " ,(~a (get-key-solution req)) " key on your keyboard.") + (noscript (p "JavaScript is required for the captcha. Sorry!")) + (div (@ (id "captcha-area"))))) + ,(application-footer #f)))))))) (when (config-true? 'debug) (xexp->html body)) (response/output diff --git a/static/jsonp.js b/static/jsonp.js index 99c4885..2e2c2db 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -88,7 +88,10 @@ async function cont() { xhr.addEventListener("load", () => { console.log(xhr) + // check for errors if (xhr.status === 500) return fetch(pkg.url, pkg.init).then(res => res.text()).then(error) + // check for captcha screen + if (xhr.responseXML.head.querySelector('script[src*="captcha.js"]')) return location.reload() // page -> #content const imported = document.importNode(xhr.responseXML.getElementById("content"), true) document.getElementById("content").replaceWith(imported) From 8d3024b2011e96e3a610f6b1cec0c009a9fbb5f3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 02:05:51 +1300 Subject: [PATCH 17/30] error message if script blocked --- src/page-captcha.rkt | 2 +- src/page-wiki-jsonp.rkt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt index df4f5bb..dcbf818 100644 --- a/src/page-captcha.rkt +++ b/src/page-captcha.rkt @@ -151,7 +151,7 @@ (when (config-true? 'captcha::log) (printf "captcha pass - key ~a [~a]~n" x (get-ip req))) - (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10))))] + (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10) #:http-only? #t #:secure? #t)))] [(= y 0) (when (config-true? 'captcha::log) (printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))] diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt index ada846e..55c98f5 100644 --- a/src/page-wiki-jsonp.rkt +++ b/src/page-wiki-jsonp.rkt @@ -53,8 +53,8 @@ var jsonpData = {} var proxy = new Proxy(jsonpData, {get(obj, prop) { return value => obj[prop] = value }}) END ) - (script (@ (async) (src ,wiki-page-script-url))) - (script (@ (async) (src ,siteinfo-script-url))) + (script (@ (async) (src ,wiki-page-script-url) (onerror "proxy.wikipage({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})"))) + (script (@ (async) (src ,siteinfo-script-url) (onerror "proxy.siteinfo({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})"))) (script (@ (type "module") (src ,(get-static-url "jsonp.js"))))) #:req req #:source-url source-url From 968ec1962c53f395bddcc3728ea4f82b93847d62 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 02:30:27 +1300 Subject: [PATCH 18/30] undertale wikis are independent now --- src/extwiki-data.rkt | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index a8ee159..fe6ea93 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -92,6 +92,16 @@ (λ (props) '())) + 'Undertale + (extwiki-group^ + "Undertale" + '(("Reasons and community vote" . "https://undertale.fandom.com/f/p/4400000000000078624") + ("Moving announcement" . "https://undertale.fandom.com/f/p/4400000000000078715") + ("About the new wiki" . "https://undertale.wiki/w/Undertale_Wiki:About")) + (λ (props) + '((p "Following a community vote on March 26, 2025, most of the wiki's editors have moved to a new host. As a result, this wiki's content may be outdated or incorrect.") + (p "The Undertale & Deltarune Wikis voted to move off their previous host, Fandom. Their reasons included bad user experience, lack of editorial freedom, and disagreement with the sporadic AI Quick Answers that were visible to logged off readers on August 19, 2023. Both wikis moved, and are now independently hosted by the wikis' staff.")))) + 'empty (extwiki-group^ "Misc" @@ -533,6 +543,24 @@ (p "Please stop using the abandoned copy of Rain Wiki on Fandom. Fandom is still \"training\" a generator which adds procedurally-generated bullshit to articles, with no way for users to remove or correct it, and they're demanding volunteer wiki admins waste time \"vetting\" the procedurally-generated BS for accuracy. As Jocelyn herself said, \"fuck Fandom forever.\"") (p "If you are interested, please add more articles related to other Rainverse stories.")))) + (extwiki^ + '("undertale") 'default + 'Undertale + "Undertale Wiki" + "https://undertale.wiki/w/Undertale_Wiki" + "https://static.wikia.nocookie.net/undertale/images/e/e6/Site-logo.png/revision/latest?cb=20220717174821" + (λ (props) + `())) + + (extwiki^ + '("deltarune") 'default + 'Undertale + "Deltarune Wiki" + "https://deltarune.wiki/w/Deltarune_Wiki" + "https://deltarune.wiki/images/Deltarune_Wiki_logo.png?cb=cvvhwg&h=thumb.php&f=Deltarune_Wiki_logo.png" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) @@ -569,6 +597,7 @@ (extwiki^ '("totherescue") 'empty 'empty "To The Rescue!" "https://totherescue.wiki.gg/wiki/To_The_Rescue%21_Wiki" #f #f) (extwiki^ '("touhou") 'empty 'empty "Touhou Wiki" "https://en.touhouwiki.net/wiki/Touhou_Wiki" #f #f) (extwiki^ '("undermine") 'empty 'empty "Official UnderMine Wiki" "https://undermine.wiki.gg/wiki/UnderMine_Wiki" #f #f) + (extwiki^ '("undertaleyellow") 'empty 'empty "Undertale Yellow Wiki" "https://undertaleyellow.wiki.gg/wiki/Undetale_Yellow_Wiki" #f #f) (extwiki^ '("westofloathing" "loathing") 'empty 'empty "Wiki of Loathing" "https://loathing.wiki.gg/wiki/Wiki_of_Loathing" #f #f) (extwiki^ '("willyousnail") 'empty 'empty "Official Will You Snail Wiki" "https://willyousnail.wiki.gg/wiki/Will_You_Snail_Wiki" #f #f) (extwiki^ '("yumenikki" "yume-nikki-dream-diary") 'empty 'empty "Yume Wiki" "https://yume.wiki/Main_Page" #f #f))) From 94cb54a712044eb3419b44df2c8c106abe31441c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 13 Nov 2025 02:40:11 +1300 Subject: [PATCH 19/30] don't squeeze text alongside infoboxes --- static/main.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/static/main.css b/static/main.css index e89008e..101b4cb 100644 --- a/static/main.css +++ b/static/main.css @@ -500,6 +500,10 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ .page-title { color: var(--theme-body-text-color); } + /* don't squeeze text alongside infoboxes */ + .infobox, .portable-infobox { + float: none; + } } /* ***** From 7b2f96eb03963083d9827d192c7cfe8f136b9859 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 15 Nov 2025 14:28:09 +1300 Subject: [PATCH 20/30] jump to fragment --- static/jsonp.js | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/static/jsonp.js b/static/jsonp.js index 2e2c2db..7997045 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -34,20 +34,6 @@ window.proxy = new Proxy(jsonpData, { } }) -// *** Page theme - -// Try to make it a bit more accurate. Normally this is done server-side by sending a `theme=x` cookie with the parse request. But that's not possible in jsonp mode, and there's no equivalent URL query string to set the theme. -// Helps on Minecraft wiki. Might not be complete on some wikis. -cookieStore.get("theme").then(cookie => { - if (cookie && cookie.value !== "default") { - document.body.classList.remove("theme-fandomdesktop-light") - document.body.classList.remove("theme-fandomdesktop-dark") - document.body.removeAttribute("data-theme") - document.body.classList.add(`theme-fandomdesktop-${cookie.value}`) - document.body.setAttribute("data-theme", cookie.value) - } -}) - // *** Data upload and download async function cont() { @@ -95,8 +81,21 @@ async function cont() { // page -> #content const imported = document.importNode(xhr.responseXML.getElementById("content"), true) document.getElementById("content").replaceWith(imported) + // page theme + // Try to make it a bit more accurate. Normally this is done server-side by sending a `theme=x` cookie with the parse request. But that's not possible in jsonp mode, and there's no equivalent URL query string to set the theme. + // Helps on Minecraft wiki. Might not be complete on some wikis. + cookieStore.get("theme").then(cookie => { + if (cookie && cookie.value !== "default") { + document.body.classList.remove("theme-fandomdesktop-light") + document.body.classList.remove("theme-fandomdesktop-dark") + document.body.removeAttribute("data-theme") + document.body.classList.add(`theme-fandomdesktop-${cookie.value}`) + document.body.setAttribute("data-theme", cookie.value) + } + }) // document.title = xhr.responseXML.title + document.body.className = xhr.responseXML.body.className for (const e of xhr.responseXML.head.children) { const alreadyImported = [...document.querySelectorAll("link[href]")].map(e => e.href) if (e.tagName === "LINK" && !alreadyImported.includes(e.href)) { @@ -104,7 +103,11 @@ async function cont() { document.head.appendChild(imported) } } - // Redirects + // scroll + if (location.hash) { + document.getElementById(location.hash.slice(1)).scrollIntoView({behavior: "instant"}) + } + // redirects const redirectTo = document.querySelector("#content .redirectMsg a") if (redirectTo) { redirectTo.click() From b7fe1807909a37149e7864b9c3c25df0f688cee0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 17 Nov 2025 15:18:54 +1300 Subject: [PATCH 21/30] Add JSONP support on other endpoints Refactored the code to define and process endpoints rather than doing it imperatively, which avoided a lot of duplicated code. --- breezewiki.rkt | 4 - dist.rkt | 5 +- lib/tree-updater.rkt | 2 +- src/dispatcher-tree.rkt | 6 +- src/endpoints.rkt | 248 +++++++++++++++++++++++++++++++++ src/page-category.rkt | 101 +++++++------- src/page-file.rkt | 1 + src/page-search.rkt | 11 +- src/page-wiki-jsonp.rkt | 72 ---------- src/page-wiki.rkt | 200 ++++++++++++++------------ src/search-provider-fandom.rkt | 128 +++++++++++------ static/jsonp.js | 22 +-- 12 files changed, 518 insertions(+), 282 deletions(-) create mode 100644 src/endpoints.rkt delete mode 100644 src/page-wiki-jsonp.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index 17b619b..6b3d5eb 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -27,9 +27,7 @@ (require-reloadable "src/page-static-archive.rkt" page-static-archive) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) -(require-reloadable "src/page-wiki.rkt" page-wiki-with-data) (require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline) -(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp) (require-reloadable "src/page-file.rkt" page-file) (reload!) @@ -59,9 +57,7 @@ page-set-user-settings page-static-archive page-wiki - page-wiki-with-data page-wiki-offline - page-wiki-jsonp page-file redirect-wiki-home static-dispatcher diff --git a/dist.rkt b/dist.rkt index 491ca80..1e7096e 100644 --- a/dist.rkt +++ b/dist.rkt @@ -16,9 +16,8 @@ (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-static-archive.rkt" page-static-archive)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) -(require (only-in "src/page-wiki.rkt" page-wiki page-wiki-with-data)) +(require (only-in "src/page-wiki.rkt" page-wiki)) (require (only-in "src/page-wiki-offline.rkt" page-wiki-offline)) -(require (only-in "src/page-wiki-jsonp.rkt" page-wiki-jsonp)) (require (only-in "src/page-file.rkt" page-file)) (serve/launch/wait @@ -43,8 +42,6 @@ page-static-archive page-wiki page-wiki-offline - page-wiki-with-data - page-wiki-jsonp page-file redirect-wiki-home static-dispatcher diff --git a/lib/tree-updater.rkt b/lib/tree-updater.rkt index f941285..c1af714 100644 --- a/lib/tree-updater.rkt +++ b/lib/tree-updater.rkt @@ -12,7 +12,7 @@ update-tree-wiki) (define (preprocess-html-wiki html) - (regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:
  • |[ \t]*?

    (.*?)

    )" + (regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:[ \t]*)?(?:
  • |[ \t]*?

    (.*?)

    )" html (λ (whole first-tag [contents #f]) (if (eq? (string-ref whole 1) #\f) ;; figcaption (string-append first-tag "" contents "") diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 0bbf8c8..fb9fa1f 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -55,6 +55,7 @@ (define tree (sequencer:make subdomain-dispatcher + (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works)) (pathprocedure:make "/" (page ds page-home)) (filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher)) (filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image))) @@ -65,16 +66,11 @@ (pathprocedure:make "/proxy" (page ds page-proxy)) (pathprocedure:make "/search" (page ds page-global-search)) (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings)) - (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works)) - (pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category))) (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file))) (if (config-true? 'feature_offline::enabled) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) - (if (config-true? 'feature_jsonp::enabled) - (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp))) - (λ (_conn _req) (next-dispatcher))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home))) diff --git a/src/endpoints.rkt b/src/endpoints.rkt new file mode 100644 index 0000000..7b2115a --- /dev/null +++ b/src/endpoints.rkt @@ -0,0 +1,248 @@ +#lang racket/base +(require racket/format + racket/match + racket/string + ; libs + (prefix-in easy: net/http-easy) + json + ; html libs + html-writing + ; web server libs + web-server/http + web-server/dispatchers/dispatch + ; my libs + "application-globals.rkt" + "config.rkt" + "data.rkt" + "fandom-request.rkt" + "static-data.rkt" + "../lib/archive-file-mappings.rkt" + "../lib/thread-utils.rkt" + "../lib/url-utils.rkt" + "../lib/xexpr-utils.rkt") +(require (for-syntax racket/base syntax/parse)) + +(provide + define-endpoint + define-standard-handler + define-post-data-handler + define-jsonp-handler + make-switch-handler + (all-from-out racket/format + racket/match + json + net/http-easy + web-server/dispatchers/dispatch + "../lib/thread-utils.rkt" + "../lib/archive-file-mappings.rkt" + "config.rkt" + "data.rkt" + "fandom-request.rkt" + "static-data.rkt")) + + +(define-for-syntax (fix-scopes here orig stx) + (define remover (make-syntax-delta-introducer here #f)) + (define introducer (make-syntax-delta-introducer orig #f)) + (introducer (remover stx 'remove))) + + +(define-syntax (define-endpoint stx) + (syntax-parse stx + [(_ name + (~and (~seq all ...) + (~seq + ((~datum variables) variable ...) + ((~datum endpoints) endpoint ...) + ((~datum render) render ...)))) + #'(define-syntax name #'(all ...))])) + + +(define ((make-switch-handler #:standard standard #:jsonp jsonp #:post post) req) + (cond + [(equal? (request-method req) #"POST") + (post req)] + [(config-true? 'feature_jsonp::enabled) + (jsonp req)] + [else + (standard req)])) + + +(define-syntax (define-standard-handler stx) + (syntax-parse stx + [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...) + #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref))) + (syntax-parse #'endpoint-syntax + #:context 'endpoint-data + [(((~datum variables) variable ...) + ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...) + ((~datum render) render ...)) + #:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))] + [params (syntax-e #'((endpoint-param ...) ...))]) + (with-syntax ([params (append (syntax-e params) #;'(("callback" . "proxy.wikipage")))]) + (if (eq? (syntax-e id) 'siteinfo) + #'(λ () + (siteinfo-fetch wikiname)) + #'(λ () + (fandom-get-api + wikiname + `params + #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))))) + #:with endpoint-intermediates->values (for/list ([id (syntax-e #'(endpoint-id ...))]) + (with-syntax ([id id]) + (if (eq? (syntax-e #'id) 'siteinfo) + #'id + #'(easy:response-json id)))) + + (fix-scopes + #'here stx + #'(define name-args + (response-handler + (let/cc k + variable ... + (define user-cookies (user-cookies-getter req)) + (define-values (endpoint-id ...) + (let-values ([(endpoint-id ...) (thread-values (~@ . endpoint-getters))]) + (for ([response (list endpoint-id ...)] + #:when (easy:response? response)) + (match (easy:response-status-code response) + [404 + (next-dispatcher)] + [(or 403 406) + (define body + (generate-wiki-page + `(div + (p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.") + (p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.") + (p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom.")))) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:siteinfo siteinfo)) + (k (response/output + #:code 503 + #:headers (build-headers + always-headers + (header #"Retry-After" #"30") + (header #"Cache-Control" #"max-age=30, public") + (header #"Refresh" #"35")) + (λ (out) + (write-html body out))))] + [(not 200) + (k (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" + wikiname + path + (easy:response-status-code response) + (easy:response-body response)))] + [_ (void)])) + (values (~@ . endpoint-intermediates->values)))) + fn-body ... + render ...))))])])) + + +(define-syntax (define-post-data-handler stx) + (syntax-parse stx + [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...) + #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref))) + (syntax-parse #'endpoint-syntax + #:context 'endpoint-data + [(((~datum variables) variable ...) + ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...) + ((~datum render) render ...)) + #:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))] + [params (syntax-e #'((endpoint-param ...) ...))]) + (with-syntax ([id id]) + (if (eq? (syntax-e #'id) 'siteinfo) + #'(data->siteinfo (hash-ref post-data 'id)) + #'(hash-ref post-data 'id)))) + + (fix-scopes + #'here stx + #'(define name-args + (response-handler + (let/cc k + (define (k-validation-error message) + (k (response/jsexpr + #:code 400 + #:headers always-headers + `#hasheq((error . + #hasheq((code . "breezewiki") + (info . ,message))))))) + (define post-data/bytes (request-post-data/raw req)) + (when (not post-data/bytes) + (k-validation-error "POST requests only, please.")) + + (define origin-header + (or (headers-assq* #"origin" (request-headers/raw req)) + (headers-assq* #"referer" (request-headers/raw req)))) + (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin)))) + (k-validation-error "Origin/Referer header failed validation - cross-origin requests are not allowed here")) + + (define post-data/string (bytes->string/utf-8 post-data/bytes)) + (define post-data (string->jsexpr post-data/string)) + (define-values (endpoint-id ...) (values (~@ . endpoint-getters))) + (define wikiname (hash-ref post-data 'wikiname)) + (define path (hash-ref post-data 'path)) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) + fn-body ... + render ...))))])])) + + +(define-syntax (define-jsonp-handler stx) + (syntax-parse stx + [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...) + #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref))) + (syntax-parse #'endpoint-syntax + #:context 'endpoint-data + [(((~datum variables) variable ...) + ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...) + ((~datum render) render ...)) + #:with endpoint-scripts (for/list ([id (syntax-e #'(endpoint-id ...))] + [params (syntax-e #'((endpoint-param ...) ...))]) + (with-syntax ([id id] + [params (append (syntax-e params) + `(("callback" . ,(format "proxy.~a" (syntax-e id)))))]) + #'(script (@ (async) + (src ,(format "https://~a.fandom.com/api.php?~a" + wikiname + (params->query `params))) + (data-jsonp-var ,(format "~a" 'id)) + (onerror ,(format "proxy.~a({error: {code: 'disconnected', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})" 'id)))))) + + (fix-scopes + #'here stx + #'(define name-args + (response-handler + (let/cc k + variable ... + fn-body ... + + (define body + (generate-wiki-page + `(div + (noscript "You have to enable JavaScript to load wiki pages. Sorry!") + (div (@ (id "loading"))) + (div (@ (id "progress-bar") (style "margin-bottom: 50vh")) + (progress)) + (script #< obj[prop] = value }}) +END + ) + endpoint-scripts + (script (@ (type "module") (src ,(get-static-url "jsonp.js"))))) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:siteinfo siteinfo-default + #:path path + #:jsonp #t)) + (when (config-true? 'debug) + (xexp->html body)) + (response/output + #:code 200 + #:headers always-headers + (λ (out) + (write-html body out)))))))])])) diff --git a/src/page-category.rkt b/src/page-category.rkt index e1fe659..8913a81 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -1,8 +1,5 @@ #lang racket/base -(require racket/dict - racket/list - racket/match - racket/string +(require racket/string (prefix-in easy: net/http-easy) ; html libs html-parsing @@ -11,14 +8,9 @@ net/url web-server/http (only-in web-server/dispatchers/dispatch next-dispatcher) - #;(only-in web-server/http/redirect redirect-to) "application-globals.rkt" - "config.rkt" - "data.rkt" - "fandom-request.rkt" - "page-wiki.rkt" - "../lib/syntax.rkt" - "../lib/thread-utils.rkt" + "endpoints.rkt" + "../lib/tree-updater.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt") @@ -63,47 +55,47 @@ ,title))) members)))))) -(define (page-category req) - (response-handler - (define wikiname (path/param-path (first (url-path (request-uri req))))) - (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-category)) - (define-values (members-data page-data siteinfo) - (thread-values - (λ () - (easy:response-json - (fandom-get-api - wikiname - `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (λ () - (easy:response-json - (fandom-get-api - wikiname - `(("action" . "parse") - ("page" . ,prefixed-category) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (λ () - (siteinfo-fetch wikiname)))) - - (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) - (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) +(define-endpoint + category-endpoint + [variables + (define wikiname (path/param-path (car (url-path (request-uri req))))) + (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) + (define title (url-segments->guess-title segments)) + (define path (string-join (cdr segments) "/")) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-category))] + [endpoints + (membersdata + (("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))) + (pagedata + (("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))) + (siteinfo + (("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2")))] + [render + (define page-html (preprocess-html-wiki (jp "/parse/text" pagedata ""))) (define page (html->xexp page-html)) - (define head-data ((head-data-getter wikiname) page-data)) + (define head-data ((head-data-getter wikiname) pagedata)) (define body (generate-results-page #:req req #:source-url source-url #:wikiname wikiname - #:title title - #:members-data members-data + #:title (preprocess-html-wiki (jp "/parse/title" pagedata prefixed-category)) + #:members-data membersdata #:page page #:head-data head-data #:siteinfo siteinfo)) @@ -116,7 +108,22 @@ #:code 200 #:headers (build-headers always-headers) (λ (out) - (write-html body out))))) + (write-html body out)))]) + +(define-standard-handler (page-category-standard req) + #'category-endpoint) + +(define-jsonp-handler (page-category-jsonp req) + #'category-endpoint) + +(define-post-data-handler (page-category-with-data req) + #'category-endpoint + (define prefixed-category path)) + +(define page-category (make-switch-handler #:standard page-category-standard + #:jsonp page-category-jsonp + #:post page-category-with-data)) + (module+ test (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") (generate-results-page diff --git a/src/page-file.rkt b/src/page-file.rkt index 5151f1d..78dee37 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -18,6 +18,7 @@ "fandom-request.rkt" "page-wiki.rkt" "../lib/syntax.rkt" + "../lib/tree-updater.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt") diff --git a/src/page-search.rkt b/src/page-search.rkt index 39f361a..9eec557 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -23,8 +23,7 @@ page-search) (define search-providers - (hash "fandom" search-fandom - "solr" search-solr)) + (hash "solr" search-solr)) ;; this takes the info we gathered from fandom and makes the big fat x-expression page (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f]) @@ -40,7 +39,7 @@ results-content)) ;; will be called when the web browser asks to load the page -(define (page-search req) +(define (page-search-solr req) ;; this just means, catch any errors and display them in the browser. it's a function somewhere else (response-handler ;; the URL will look like "/minecraft/wiki/Special:Search?q=Spawner" @@ -84,3 +83,9 @@ (λ (out) (write-html body out))))) + + +(define (page-search req) + (if (equal? (config-get 'feature_offline::search) "fandom") + (page-search-fandom req) + (page-search-solr req))) \ No newline at end of file diff --git a/src/page-wiki-jsonp.rkt b/src/page-wiki-jsonp.rkt deleted file mode 100644 index 55c98f5..0000000 --- a/src/page-wiki-jsonp.rkt +++ /dev/null @@ -1,72 +0,0 @@ -#lang racket/base -(require racket/list - racket/string - web-server/http - net/url-structs - html-writing - "application-globals.rkt" - "data.rkt" - "config.rkt" - "../lib/url-utils.rkt" - "../lib/xexpr-utils.rkt" - "../lib/archive-file-mappings.rkt" - "static-data.rkt") - -(provide - page-wiki-jsonp) - -(define (page-wiki-jsonp req) - (response-handler - (define wikiname (path/param-path (first (url-path (request-uri req))))) - (define segments (map path/param-path (cdr (url-path (request-uri req))))) - (define path (string-join (cdr segments) "/")) - (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) - - (define wiki-page-script-url - (format "https://~a.fandom.com/api.php?~a" - wikiname - (params->query `(("action" . "parse") - ("page" . ,path) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json") - ("callback" . "proxy.wikipage"))))) - (define siteinfo-script-url - (format "https://~a.fandom.com/api.php?~a" - wikiname - (params->query `(("action" . "query") - ("meta" . "siteinfo") - ("siprop" . "general|rightsinfo") - ("format" . "json") - ("formatversion" . "2") - ("callback" . "proxy.siteinfo"))))) - - (define body - (generate-wiki-page - `(div - (noscript "You have to enable JavaScript to load wiki pages. Sorry!") - (div (@ (id "loading"))) - (div (@ (id "progress-bar") (style "margin-bottom: 50vh")) - (progress)) - (script #< obj[prop] = value }}) -END - ) - (script (@ (async) (src ,wiki-page-script-url) (onerror "proxy.wikipage({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})"))) - (script (@ (async) (src ,siteinfo-script-url) (onerror "proxy.siteinfo({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})"))) - (script (@ (type "module") (src ,(get-static-url "jsonp.js"))))) - #:req req - #:source-url source-url - #:wikiname wikiname - #:title (url-segments->guess-title segments) - #:siteinfo siteinfo-default - #:path path - #:jsonp #t)) - (when (config-true? 'debug) - (xexp->html body)) - (response/output - #:code 200 - #:headers always-headers - (λ (out) - (write-html body out))))) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 1caf41d..d87cb96 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -1,12 +1,6 @@ #lang racket/base (require racket/dict - racket/function - racket/list - racket/match racket/string - ; libs - (prefix-in easy: net/http-easy) - json ; html libs "../lib/html-parsing/main.rkt" html-writing @@ -16,27 +10,121 @@ web-server/dispatchers/dispatch ; my libs "application-globals.rkt" - "config.rkt" - "data.rkt" - "fandom-request.rkt" - "../lib/archive-file-mappings.rkt" - "../lib/thread-utils.rkt" + "endpoints.rkt" "../lib/tree-updater.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt") +(require (for-syntax racket/base syntax/parse)) (provide - ; used by the web server - page-wiki - page-wiki-with-data - ; used by page-category, and similar pages that are partially wiki pages - update-tree-wiki - preprocess-html-wiki) + page-wiki) -(module+ test - (require rackunit)) +(define-endpoint + wiki-endpoint + [variables + (define wikiname (path/param-path (car (url-path (request-uri req))))) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) + (define title (url-segments->guess-title segments)) + (define path (string-join (cdr segments) "/")) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))] + [endpoints + (wikipage (("action" . "parse") + ("page" . ,path) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))) + (siteinfo (("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2")))] + [render + (define page-html (preprocess-html-wiki (jp "/parse/text" wikipage ""))) + (define page (html->xexp page-html)) + (define head-data ((head-data-getter wikiname) wikipage)) + (define body + (generate-wiki-page + (update-tree-wiki page wikiname) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title (jp "/parse/title" wikipage "") + #:head-data head-data + #:siteinfo siteinfo)) + (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) + (define html (xexp->html-bytes body)) + (define headers + (build-headers + always-headers + ; redirect-query-parameter: only the string "no" is significant: + ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 + (when (and redirect-msg-a + (not (equal? redirect-query-parameter "no"))) + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] + [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) + (header #"Refresh" value))))) + (response/full + 200 + #"OK" + (current-seconds) + #"text/html; charset=utf-8" + headers + (list html))]) -(define (page-wiki req) +(define-standard-handler (page-wiki-standard req) + #'wiki-endpoint + (when (equal? "missingtitle" (jp "/error/code" wikipage #f)) + (next-dispatcher))) + +(define-jsonp-handler (page-wiki-jsonp req) + #'wiki-endpoint) + +(define-post-data-handler (page-wiki-with-data req) + #'wiki-endpoint) + +(define page-wiki (make-switch-handler #:standard page-wiki-standard + #:jsonp page-wiki-jsonp + #:post page-wiki-with-data)) + +#;(define (page-wiki-with-data req) + (response-handler + (let/cc return + (define post-data/bytes (request-post-data/raw req)) + (when (not post-data/bytes) + (k (response/jsexpr + #:code 400 + #:headers always-headers + '#hasheq((error . + #hasheq((code . "breezewiki") + (info . "POST requests only, please."))))))) + + (define origin-header + (or (headers-assq* #"origin" (request-headers/raw req)) + (headers-assq* #"referer" (request-headers/raw req)))) + (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin)))) + (k (response/jsexpr + #:code 400 + #:headers always-headers + '#hasheq((error . + #hasheq((code . "breezewiki") + (info . "Origin/Referer header failed validation - cross-origin requests are not allowed here"))))))) + + (define post-data/string (bytes->string/utf-8 post-data/bytes)) + (define post-data (string->jsexpr post-data/string)) + (define wikiname (jp "/wikiname" post-data)) + (define path (jp "/path" post-data)) + (take-json-rewrite-and-return-page + #:req req + #:wikiname wikiname + #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) + #:data (jp "/data" post-data) + #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))))) + +#;(define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) (define segments (map path/param-path (cdr (url-path (request-uri req))))) (define user-cookies (user-cookies-getter req)) @@ -95,75 +183,3 @@ (easy:response-status-code dest-res) (easy:response-body dest-res)))])) -(define (page-wiki-with-data req) - (response-handler - (let/cc return - (define post-data/bytes (request-post-data/raw req)) - (when (not post-data/bytes) - (return (response/jsexpr - #:code 400 - #:headers always-headers - '#hasheq((error . - #hasheq((code . "breezewiki") - (info . "POST requests only, please."))))))) - - (define origin-header - (or (headers-assq* #"origin" (request-headers/raw req)) - (headers-assq* #"referer" (request-headers/raw req)))) - (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin)))) - (return (response/jsexpr - #:code 400 - #:headers always-headers - '#hasheq((error . - #hasheq((code . "breezewiki") - (info . "Origin/Referer header failed validation - cross-origin requests are not allowed here"))))))) - - (define post-data/string (bytes->string/utf-8 post-data/bytes)) - (define post-data (string->jsexpr post-data/string)) - (define wikiname (jp "/wikiname" post-data)) - (define path (jp "/path" post-data)) - (take-json-rewrite-and-return-page - #:req req - #:wikiname wikiname - #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path) - #:data (jp "/data" post-data) - #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))))) - -(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo) - (define title (jp "/parse/title" data "")) - (define page-html (preprocess-html-wiki (jp "/parse/text" data ""))) - (define page (html->xexp page-html)) - (define head-data ((head-data-getter wikiname) data)) - (response-handler - (define body - (generate-wiki-page - (update-tree-wiki page wikiname) - #:req req - #:source-url source-url - #:wikiname wikiname - #:title title - #:head-data head-data - #:siteinfo siteinfo)) - (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) - (define redirect-msg-a (if redirect-msg - ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) - #f)) - (define html (xexp->html-bytes body)) - (define headers - (build-headers - always-headers - ; redirect-query-parameter: only the string "no" is significant: - ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg-a - (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] - [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (header #"Refresh" value))))) - (response/full - 200 - #"OK" - (current-seconds) - #"text/html; charset=utf-8" - headers - (list html)))) diff --git a/src/search-provider-fandom.rkt b/src/search-provider-fandom.rkt index b8dd48f..d87c868 100644 --- a/src/search-provider-fandom.rkt +++ b/src/search-provider-fandom.rkt @@ -1,59 +1,95 @@ #lang racket/base -(require racket/string +(require racket/dict + racket/string (prefix-in easy: net/http-easy) + net/url + web-server/http + html-writing "application-globals.rkt" "config.rkt" + "endpoints.rkt" "fandom-request.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt") (provide - search-fandom) + page-search-fandom) -(module+ test - (require rackunit - "test-utils.rkt") - (define search-results-data - '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) +(define-endpoint + search-endpoint + [variables + (define wikiname (path/param-path (car (url-path (request-uri req))))) + (define params (url-query (request-uri req))) + (define query (dict-ref params 'q #f)) + (define title "Search") + (define path (format "Special:Search?~a" (params->query `(("query" . ,query) + ("search" . "internal"))))) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))] + [endpoints + (search + (("action" . "query") + ("list" . "search") + ("srsearch" . ,query) + ("formatversion" . "2") + ("format" . "json"))) + (siteinfo + (("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2")))] + [render + (define search-results (jp "/query/search" search)) + (define body + (generate-wiki-page + #:req req + #:source-url source-url + #:wikiname wikiname + #:title query + #:siteinfo siteinfo + `(div (@ (class "mw-parser-output")) + ;; header before the search results showing how many we found + (p ,(format "~a results found for " (length search-results)) + (strong ,query)) + ;; *u*nordered *l*ist of matching search results + (ul ,@(for/list ([result search-results]) + (let* ([title (jp "/title" result)] + [page-path (page-title->path title)] + [timestamp (jp "/timestamp" result)] + [wordcount (jp "/wordcount" result)] + [size (jp "/size" result)]) + ;; and make this x-expression... + `(li (@ (class "my-result")) + (a (@ (class "my-result__link") (href ,(format "/~a/wiki/~a" wikiname page-path))) ; using unquote to insert the result page URL + ,title) ; using unquote to insert the result page title + (div (@ (class "my-result__info")) ; constructing the line under the search result + "last edited " + (time (@ (datetime ,timestamp)) ,(list-ref (string-split timestamp "T") 0)) + ,(format ", ~a words, ~a kb" + wordcount + (exact->inexact (/ (round (/ size 100)) 10))))))))))) + (when (config-true? 'debug) + (xexp->html body)) + (response/output + #:code 200 + #:headers (build-headers always-headers) + (λ (out) + (write-html body out)))]) -(define (search-fandom wikiname query params) - (define res - (fandom-get-api - wikiname - `(("action" . "query") - ("list" . "search") - ("srsearch" . ,query) - ("formatversion" . "2") - ("format" . "json")))) - (define json (easy:response-json res)) - (define search-results (jp "/query/search" json)) - (generate-results-content-fandom wikiname query search-results)) -;;; generate content for display in the wiki page layout -(define (generate-results-content-fandom wikiname query search-results) - `(div (@ (class "mw-parser-output")) - ;; header before the search results showing how many we found - (p ,(format "~a results found for " (length search-results)) - (strong ,query)) - ;; *u*nordered *l*ist of matching search results - (ul ,@(for/list ([result search-results]) - (let* ([title (jp "/title" result)] - [page-path (page-title->path title)] - [timestamp (jp "/timestamp" result)] - [wordcount (jp "/wordcount" result)] - [size (jp "/size" result)]) - ;; and make this x-expression... - `(li (@ (class "my-result")) - (a (@ (class "my-result__link") (href ,(format "/~a/wiki/~a" wikiname page-path))) ; using unquote to insert the result page URL - ,title) ; using unquote to insert the result page title - (div (@ (class "my-result__info")) ; constructing the line under the search result - "last edited " - (time (@ (datetime ,timestamp)) ,(list-ref (string-split timestamp "T") 0)) - ,(format ", ~a words, ~a kb" - wordcount - (exact->inexact (/ (round (/ size 100)) 10)))))))))) +(define-standard-handler (page-search-standard req) + #'search-endpoint) -(module+ test - (parameterize ([(config-parameter 'feature_offline::only) "false"]) - (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") - (generate-results-content-fandom "test" "Gacha" search-results-data)))))) +(define-jsonp-handler (page-search-jsonp req) + #'search-endpoint) + +(define-post-data-handler (page-search-with-data req) + #'search-endpoint + (define params (url-query (request-uri req))) + (define query (dict-ref params 'q #f)) + (define title "Search")) + +(define page-search-fandom + (make-switch-handler #:standard page-search-standard + #:jsonp page-search-jsonp + #:post page-search-with-data)) diff --git a/static/jsonp.js b/static/jsonp.js index 7997045..efd96a1 100644 --- a/static/jsonp.js +++ b/static/jsonp.js @@ -37,23 +37,29 @@ window.proxy = new Proxy(jsonpData, { // *** Data upload and download async function cont() { - if (jsonpData.wikipage?.error) return error(jsonpData.wikipage) - if (jsonpData.siteinfo?.error) return error(jsonpData.siteinfo) - if (!(jsonpData.wikipage && jsonpData.siteinfo)) return + // Check for errors + for (const v of Object.values(jsonpData)) { + if (v.error) return error(v) + } + + // Check for completion + const dependencies = [...document.querySelectorAll("[data-jsonp-var]")].map(e => e.getAttribute("data-jsonp-var")) + for (const d of dependencies) { + if (!(d in jsonpData)) return + } const xhr = new XMLHttpRequest(); const uploadFraction = 0.7 const pkg = { - url: "/api/render/wiki", + url: location.href, init: { method: "POST", body: JSON.stringify({ - data: jsonpData.wikipage, - siteinfo: jsonpData.siteinfo, wikiname: BWData.wikiname, - path: BWData.path + path: BWData.path, + ...jsonpData }) } } @@ -127,7 +133,7 @@ function error(data) { if (typeof data === "string") { render(html`

    BreezeWiki ran into an error on this page.

    Try reloading the page.

    If this keeps happening, you could send a public bug report. Please include the following information:

    URL: ${window.location.href}${"\n"}${data}
    `, eContent) } else if (data.error.code === "missingtitle") { - render(html`

    This page doesn't exist on Fandom.

    Return to the homepage?

    `, eContent) + render(html`

    This page doesn't exist on Fandom.

    Return to the wiki's main page

    `, eContent) } else { render(html`

    BreezeWiki wasn't able to load this page.

    ${data.error.code}: ${data.error.info}

    `, eContent) } From 93369aa39b4fb0fd51911db8d59b9eb4d8211fd4 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 19 Nov 2025 17:07:40 +1300 Subject: [PATCH 22/30] make the captcha red --- src/page-captcha.rkt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/page-captcha.rkt b/src/page-captcha.rkt index dcbf818..6b410d4 100644 --- a/src/page-captcha.rkt +++ b/src/page-captcha.rkt @@ -53,7 +53,7 @@ (define diameter 35) (define font (make-object font% 12 'system)) (define msg "I'm not a robot!") -(define checkbox (filled-ellipse diameter diameter #:color "Pale Goldenrod")) +(define checkbox (filled-ellipse diameter diameter #:color "Red")) (define assembly (frame (inset @@ -81,6 +81,7 @@ (printf "captcha skip - via ~a [~a] - ~v~n" method (get-ip req) (url->string (request-uri req)))) (next-dispatcher)] [_ (void)])) + (response-handler (define body `(*TOP* From a26fe3cd13172abb7d66e5ee0dd5c56cface740d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 22 Nov 2025 14:46:22 +1300 Subject: [PATCH 23/30] remove genai quick answers --- lib/tree-updater.rkt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/tree-updater.rkt b/lib/tree-updater.rkt index c1af714..e911121 100644 --- a/lib/tree-updater.rkt +++ b/lib/tree-updater.rkt @@ -274,6 +274,9 @@ ; remove gamespot reviews/ads [(has-class? "reviews" attributes) return-no-element] + ; remove genai quick answers + [(has-class? "trfc161" attributes) + return-no-element] ; remove customcollapsible customtoggle buttons - sample: warframe/wiki/Amp_(Ability) [(and (dict-has-key? attributes 'class) (regexp-match? #rx"^mw-customtoggle-[^ ]* button-c$" (car (dict-ref attributes 'class)))) return-no-element] From fb48224f6bccde47ecd9b2d98bcd712afe87b29a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Nov 2025 01:55:57 +1300 Subject: [PATCH 24/30] update file page for jsonp --- info.rkt | 2 +- lib/make-json.rkt | 83 ++++++++++++++++ src/page-file.rkt | 242 +++++++++++++++++++++++++++++++--------------- static/main.css | 3 + 4 files changed, 252 insertions(+), 78 deletions(-) create mode 100644 lib/make-json.rkt diff --git a/info.rkt b/info.rkt index c290d5b..40c9399 100644 --- a/info.rkt +++ b/info.rkt @@ -1,3 +1,3 @@ #lang info -(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "typed-ini-lib" "memo" "net-cookies-lib" "db")) +(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "typed-ini-lib" "memo" "net-cookies-lib" "db" "sequence-tools-lib")) diff --git a/lib/make-json.rkt b/lib/make-json.rkt new file mode 100644 index 0000000..9a6a557 --- /dev/null +++ b/lib/make-json.rkt @@ -0,0 +1,83 @@ +#lang racket/base +(require racket/list/grouping + racket/match + racket/syntax) + +(provide make-json) + +(module+ test + (require rackunit json) + (define sample + `(: continue + (: iistart "2022-01-23T03:44:17Z" + fucontinue "455" + continue "||") + query + (: pages + (: 198 + (: pageid 198 + ns 6 + title "File:Rainbow Flag1.svg" + imageinfo + ((: timestamp "2025-03-10T07:24:50Z" + user "DogeMcMeow")) + fileusage + ((: pageid 191 + ns 0 + title "Gay") + (: pageid 215 + ns 0 + title "LGBTQIA+")))))))) + +(define (make-json data) + (match data + [(list ': kvs ...) + (for/fold ([h (hasheq)]) + ([kv (windows 2 2 kvs)]) + (match-define (list raw-k v) kv) + (define k (format-symbol "~a" raw-k)) + (hash-set h k (make-json v)))] + [(list x ...) + (map make-json x)] + [x + x])) + +(module+ test + (check-equal? (make-json sample) + (string->jsexpr #<\n\n\n\n\n") + langlinks () + categories () + links () + templates () + images () + externallinks () + sections () + parsewarnings () + displaytitle "File:Sailor Cinnamoroll.jpg" + iwlinks () + properties ())))) + (define test-imageinfo-outer + (make-json + `(: continue + (: iistart "2022-01-23T03:44:17Z" + fucontinue "455" + continue "||") + query + (: pages + ((: pageid 198 + ns 6 + title "File:Rainbow Flag1.svg" + imageinfo + ((: timestamp "2025-03-10T07:24:50Z" + user "DogeMcMeow" + url "https://static.wikia.nocookie.net/lgbtqia-sandbox/images/f/f8/Rainbow_Flag1.svg/revision/latest?cb=20250310072450" + descriptionurl "https://lgbtqia.fandom.com/wiki/File:Rainbow_Flag1.svg" + descriptionshorturl "https://lgbtqia.fandom.com/index.php?curid=198" + mime "image/svg+xml" + mediatype "DRAWING")) + fileusage + ((: pageid 191 + ns 0 + title "Gay") + (: pageid 215 + ns 0 + title "LGBTQIA+"))))))))) -(define (url-content-type url) - (define dest-res (easy:head url)) - (easy:response-headers-ref dest-res 'content-type)) +;; https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/core/+/refs/heads/REL1_44/includes/libs/mime/defines.php +;; (define possible-mediatypes '("UNKNOWN" "BITMAP" "DRAWING" "AUDIO" "VIDEO" "MULTIMEDIA" "OFFICE" "TEXT" "EXECUTABLE" "ARCHIVE" "3D")) -(define (get-media-html url content-type) +(define (get-media-html url imageinfo-inner) (define maybe-proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) - (cond - [(eq? content-type #f) `""] - [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,maybe-proxied-url)))] - [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) - `(audio (@ (src ,maybe-proxied-url) (controls)))] - [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,maybe-proxied-url) (controls)))] - [else `""])) + (define mediatype (jp "/mediatype" imageinfo-inner #f)) + (case mediatype + [("BITMAP" "DRAWING") + (match imageinfo-inner + [(hash* ['width width] ['height height]) + `(img (@ (src ,maybe-proxied-url) (width ,(~a width)) (height ,(~a height))))] + [else + `(img (@ (src ,maybe-proxied-url)))])] + [("AUDIO") `(audio (@ (src ,maybe-proxied-url) (controls)))] + [("VIDEO") `(video (@ (src ,maybe-proxied-url) (controls)))] + [else ""])) (define (generate-results-page #:req req #:source-url source-url #:wikiname wikiname #:title title - #:media-detail media-detail - #:image-content-type image-content-type + #:wikipage wikipage + #:imageinfo imageinfo-outer #:siteinfo [siteinfo #f]) - (define video-embed-code (jp "/videoEmbedCode" media-detail "")) - (define raw-image-url (jp "/rawImageUrl" media-detail)) - (define image-url (jp "/imageUrl" media-detail raw-image-url)) - (define username (jp "/userName" media-detail)) - (define is-posted-in (jp "/isPostedIn" media-detail #f)) - (define smaller-article-list (jp "/smallerArticleList" media-detail)) - (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) - (define image-description (jp "/imageDescription" media-detail #f)) - (define maybe-proxied-raw-image-url - (if (config-true? 'strict_proxy) (u-proxy-url raw-image-url) raw-image-url)) + (define imageinfo-inner (jp "/query/pages/0" imageinfo-outer)) + (define fileusage-continues? (jp "/continue/fucontinue" imageinfo-outer #f)) + (define fileusage-titles (for/list ([page (jp "/fileusage" imageinfo-inner)]) (jp "/title" page))) + (define image-url (jp "/imageinfo/0/url" imageinfo-inner)) + (define username (jp "/imageinfo/0/user" imageinfo-inner)) + (define maybe-proxied-image-url + (if (config-true? 'strict_proxy) (u-proxy-url image-url) image-url)) (generate-wiki-page #:req req #:source-url source-url #:wikiname wikiname #:title title #:siteinfo siteinfo - `(div ,(if (non-empty-string? video-embed-code) - (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - (get-media-html image-url image-content-type)) - (p ,(if (non-empty-string? video-embed-code) - `"" - `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) - "Uploaded by " + `(div ,(get-media-html maybe-proxied-image-url (jp "/imageinfo/0" imageinfo-inner)) + (p (a (@ (href ,maybe-proxied-image-url)) "Download original file")) + ,(if (pair? (jp "/parse/sections" wikipage)) + (update-tree-wiki (html->xexp (preprocess-html-wiki (jp "/parse/text" wikipage ""))) wikiname) + ; file license info wasn't displayed in the description (example: /lgbtqia/wiki/File:Rainbow_Flag1.svg) + `(p "This file may be copyrighted. Consider licensing and fair use law before reusing it.")) + (p "Uploaded by " (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) ".") - ,(if (string? image-description) - (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) - ; file license info might be displayed in the description, example: /lgbtqia/wiki/File:Rainbow_Flag1.svg - `(p "This file is likely protected by copyright. Consider the file's license and fair use law before reusing it.")) - ,(if is-posted-in + ,(if (pair? fileusage-titles) `(p "This file is used in " - ,@(map (λ (article) - (define title (jp "/titleText" article)) - (define page-path (regexp-replace* #rx" " title "_")) - `(span ,(if (eq? (car smaller-article-list) article) "" ", ") - (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title))) - smaller-article-list) - ,(if (eq? article-list-is-smaller 1) "…" ".")) + ,@(add-between + (for/list ([title fileusage-titles] + [i (in-naturals)]) + (define page-path (regexp-replace* #rx" " title "_")) + `(a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title)) + ", ") + ,(if fileusage-continues? "…" ".")) `"")))) -(define (page-file req) +(define-endpoint + file-endpoint + [variables + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-file (path/param-path (caddr (url-path (request-uri req))))) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) + (define title (url-segments->guess-title segments)) + (define path (string-join (cdr segments) "/")) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-file))] + [endpoints + (wikipage + (("action" . "parse") + ("page" . ,prefixed-file) + ("prop" . "text|headhtml|langlinks|sections") + ("formatversion" . "2") + ("format" . "json"))) + (siteinfo + (("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2"))) + (imageinfo + (("action" . "query") + ("titles" . ,prefixed-file) + ("prop" . "imageinfo|fileusage") + ("iiprop" . "timestamp|user|canonicaltitle|url|size|mime|mediatype") + ("iilocalonly" . "true") + ("format" . "json") + ("formatversion" . "2")))] + [render + (response-handler + (define title (jp "/parse/title" wikipage prefixed-file)) + (define body + (generate-results-page #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:wikipage wikipage + #:imageinfo imageinfo + #:siteinfo siteinfo)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output #:code 200 + #:headers (build-headers always-headers) + (λ (out) (write-html body out))))]) + +(define-standard-handler (page-file-standard req) + #'file-endpoint + (when (equal? "missingtitle" (jp "/error/code" wikipage #f)) + (next-dispatcher))) + +(define-jsonp-handler (page-file-jsonp req) + #'file-endpoint) + +(define-post-data-handler (page-file-with-data req) + #'file-endpoint + (define prefixed-file path)) + +(define page-file (make-switch-handler #:standard page-file-standard + #:jsonp page-file-jsonp + #:post page-file-with-data)) + +#;(define (page-file req) (response-handler (define wikiname (path/param-path (first (url-path (request-uri req))))) (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) @@ -148,28 +236,28 @@ (λ (out) (write-html body out))))))) (module+ test (parameterize ([(config-parameter 'strict_proxy) "true"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" (make-json '(: mediatype "BITMAP"))) `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" (make-json '(: mediatype "AUDIO"))) `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") (controls))))) (parameterize ([(config-parameter 'strict_proxy) "false"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" (make-json '(: mediatype "AUDIO"))) `(audio (@ (src "https://static.wikia.nocookie.net/c") (controls)))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" (make-json '(: mediatype "VIDEO"))) `(video (@ (src "https://static.wikia.nocookie.net/d") (controls))))) - (check-equal? (get-media-html "https://example.com" "who knows") `"") - (check-equal? (get-media-html "https://example.com" #f) `"")) + (check-equal? (get-media-html "https://example.com" "who knows") "") + (check-equal? (get-media-html "https://example.com" #f) "")) (module+ test (parameterize ([(config-parameter 'strict_proxy) "true"]) (check-not-false ((query-selector - (attribute-selector 'src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fexamplefile") + (attribute-selector 'src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Flgbtqia-sandbox%2Fimages%2Ff%2Ff8%2FRainbow_Flag1.svg%2Frevision%2Flatest%3Fcb%3D20250310072450") (generate-results-page #:req test-req #:source-url "" #:wikiname "test" #:title "File:Example file" - #:media-detail test-media-detail - #:image-content-type "image/jpeg")))))) + #:wikipage test-wikipage + #:imageinfo test-imageinfo-outer)))))) diff --git a/static/main.css b/static/main.css index 101b4cb..f1fabf3 100644 --- a/static/main.css +++ b/static/main.css @@ -120,6 +120,9 @@ p { background-color: var(--theme-page-background-color); padding: 3vw; } +.fandom-community-header__background { + transform: none; /* fandom offsets this 46px by default due to their position: fixed top bar */ +} /* table of contents */ .toc { From 54eaf4186f7ab61202ea2e3f1f9b889c08fe39c6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Nov 2025 02:03:20 +1300 Subject: [PATCH 25/30] provide way to access full results page --- static/search-suggestions.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/static/search-suggestions.js b/static/search-suggestions.js index fcd5ee6..0590205 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -59,10 +59,17 @@ function Suggestion(hit) { return html`
    • ` } +function DefaultSearch({q}) { + console.log(q) + if (!q) return "" + return html`
    • ` +} + function SuggestionList() { return html`
      ${suggestions.value.map(hit => html`<${Suggestion} ...${hit} />`)} + <${DefaultSearch} q=${query.value} />
    ` } From e04ff565ee87810626341aa1f8dea6a09b842f95 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Nov 2025 03:00:44 +1300 Subject: [PATCH 26/30] add rudimentary up navigation --- lib/archive-file-mappings.rkt | 5 ++++- src/application-globals.rkt | 13 ++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/archive-file-mappings.rkt b/lib/archive-file-mappings.rkt index 03f97f5..94b4578 100644 --- a/lib/archive-file-mappings.rkt +++ b/lib/archive-file-mappings.rkt @@ -32,4 +32,7 @@ (uri-decode (regexp-replace* #rx"#" str "/"))) (define (url-segments->guess-title segments) - (regexp-replace* #rx"_" (cadr segments) " ")) + (string-join + (for/list ([s (in-list (cdr segments))]) + (regexp-replace* #rx"_" s " ")) + "/")) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 0d6b3dd..8011d01 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -235,7 +235,18 @@ (main (@ (class "page__main")) ,(extwiki-notice wikiname title req user-cookies) (div (@ (class "custom-top")) - (h1 (@ (class "page-title")) ,title) + (h1 (@ (class "page-title")) + ;; adds rudimentary "up" navigation, e.g. /minecraft/wiki/Bastion_Remnant/Structure/Blueprints/Bastion_treasure_corners_edge_middle_blueprint + ,@(let ([segments (string-split title "/")]) + (add-between + (for/list ([segment (in-list segments)] + [depth (in-naturals 1)]) + (define anti-depth (- (length segments) depth)) + (define up-href (string-join (make-list anti-depth "..") "/")) + (if (non-empty-string? up-href) + `(a (@ (href ,(format "~a/~a" up-href segment))) ,segment) + segment)) + "/"))) (nav (@ (class "sitesearch")) (form (@ (action ,(format "/~a/search" wikiname)) (class "bw-search-form") From 085fe2adc5884003ad5d5ecde29763838e287a96 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 23 Nov 2025 03:02:39 +1300 Subject: [PATCH 27/30] clean up log --- static/search-suggestions.js | 1 - 1 file changed, 1 deletion(-) diff --git a/static/search-suggestions.js b/static/search-suggestions.js index 0590205..cdcee94 100644 --- a/static/search-suggestions.js +++ b/static/search-suggestions.js @@ -60,7 +60,6 @@ function Suggestion(hit) { } function DefaultSearch({q}) { - console.log(q) if (!q) return "" return html`
    • ` } From ad7d08e176a6d4ec16959d9432f87fa6c00c8446 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 24 Nov 2025 22:00:03 +1300 Subject: [PATCH 28/30] fallback to no banner if extwiki unavailable --- src/extwiki-generic.rkt | 83 +++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/src/extwiki-generic.rkt b/src/extwiki-generic.rkt index 113b139..cd61f0b 100644 --- a/src/extwiki-generic.rkt +++ b/src/extwiki-generic.rkt @@ -85,46 +85,47 @@ (define/memoize (get-redirect-content wikiname) #:hash hash (define wiki (hash-ref wikis-hash wikiname #f)) - (cond - [wiki - (define display-name (cadr wiki)) - (define endpoint (string-append (get-api-endpoint wiki) "?action=parse&page=MediaWiki:BreezeWikiRedirect&prop=text&formatversion=2&format=json")) - (define res (get endpoint)) - (define html (jp "/parse/text" (response-json res))) - (define content ((query-selector (λ (t a c) (has-class? "mw-parser-output" a)) - (html->xexp html)))) - (define body (for/list ([p (in-producer (query-selector (λ (t a c) (eq? t 'p)) content) #f)]) p)) - (define table (parse-table ((query-selector (λ (t a c) (eq? t 'table)) content)))) - (define-values (links links-errors) (table->links table)) - (define-values (logo logo-errors) (table->logo table)) - (define construct-errors (append links-errors logo-errors)) - (λ (title) - (define go - (string-append (get-search-page wiki) - "?" - (params->query `(("search" . ,title) - ("go" . "Go"))))) - `(aside (@ (class "niwa__notice")) - (h1 (@ (class "niwa__header")) ,display-name " has its own website separate from Fandom.") - (div (@ (class "niwa__cols")) - (div (@ (class "niwa__left")) - (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,display-name " →") - ,@body - (p "This external wiki is a helpful alternative to Fandom. You should " - (a (@ (href ,go)) "check it out now!"))) - ,(if logo - `(div (@ (class "niwa__right")) - (img (@ (class "niwa__logo") (src ,logo)))) - "")) - ,(if (pair? links) - `(p (@ (class "niwa__feedback")) - ,@(add-between links " / ")) - "") - ,(if (pair? construct-errors) - `(ul - ,@(for/list ([error construct-errors]) - `(li ,error))) - "")))] - [#t #f])) + (with-handlers ([exn:fail:http-easy:timeout? (λ (e) #f)]) + (cond + [wiki + (define display-name (cadr wiki)) + (define endpoint (string-append (get-api-endpoint wiki) "?action=parse&page=MediaWiki:BreezeWikiRedirect&prop=text&formatversion=2&format=json")) + (define res (get endpoint)) + (define html (jp "/parse/text" (response-json res))) + (define content ((query-selector (λ (t a c) (has-class? "mw-parser-output" a)) + (html->xexp html)))) + (define body (for/list ([p (in-producer (query-selector (λ (t a c) (eq? t 'p)) content) #f)]) p)) + (define table (parse-table ((query-selector (λ (t a c) (eq? t 'table)) content)))) + (define-values (links links-errors) (table->links table)) + (define-values (logo logo-errors) (table->logo table)) + (define construct-errors (append links-errors logo-errors)) + (λ (title) + (define go + (string-append (get-search-page wiki) + "?" + (params->query `(("search" . ,title) + ("go" . "Go"))))) + `(aside (@ (class "niwa__notice")) + (h1 (@ (class "niwa__header")) ,display-name " has its own website separate from Fandom.") + (div (@ (class "niwa__cols")) + (div (@ (class "niwa__left")) + (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,display-name " →") + ,@body + (p "This external wiki is a helpful alternative to Fandom. You should " + (a (@ (href ,go)) "check it out now!"))) + ,(if logo + `(div (@ (class "niwa__right")) + (img (@ (class "niwa__logo") (src ,logo)))) + "")) + ,(if (pair? links) + `(p (@ (class "niwa__feedback")) + ,@(add-between links " / ")) + "") + ,(if (pair? construct-errors) + `(ul + ,@(for/list ([error construct-errors]) + `(li ,error))) + "")))] + [#t #f]))) (module+ test (check-not-false ((get-redirect-content "gallowmere") "MediEvil Wiki"))) From b061e04a3fa43f6e75e830bcb1440077aada5ab7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 31 Dec 2025 19:29:13 +1300 Subject: [PATCH 29/30] fix safety limits --- dist.rkt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist.rkt b/dist.rkt index 1e7096e..4342f7f 100644 --- a/dist.rkt +++ b/dist.rkt @@ -1,5 +1,6 @@ #lang racket/base (require web-server/servlet-dispatch + web-server/safety-limits "src/config.rkt" "src/dispatcher-tree.rkt") @@ -25,6 +26,7 @@ (if (config-true? 'debug) "127.0.0.1" #f) (config-get 'bind_host)) #:port (string->number (config-get 'port)) + #:safety-limits (make-safety-limits #:max-request-body-length (* 8 1024 1024)) (λ (quit) (dispatcher-tree ; order of these does not matter From 6afb01d3ee9ea8c7f52648d1ad7335d7841d24f8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 12 Feb 2026 00:38:34 +1300 Subject: [PATCH 30/30] Fix search colours on Just Cause 3 --- static/main.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/main.css b/static/main.css index f1fabf3..6e40f28 100644 --- a/static/main.css +++ b/static/main.css @@ -291,7 +291,6 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ margin: 0; font-size: 14px; background: white; - color: black; border: solid #808080; border-width: 0px 1px 1px; box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.5); @@ -328,6 +327,7 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ border: none; margin: 0; line-height: inherit; + color: black; background: none; font: inherit; cursor: pointer;