cadence/breezewiki
2
4
Fork 4
Code Issues Pull requests 1 Releases Wiki Activity

Add JSONP mode and captcha

Browse source 
JSONP mode is on by default. It will fetch main wiki pages in the
browser, without the server needing to make any requests. To turn it
off, add [feature_json] enabled = false to config.ini.

Captcha is off by default. It is a custom solution and is still
experimental at this stage. If you turn it on, please monitor the logs
to see how it goes! config.ini options are as follows:

[captcha]
enabled = true|false
log = true|false
ip_header = <header name set by your reverse proxy, like x-forwarded-for>
This commit is contained in:
Cadence Ember 2025-11-04 23:06:55 +13:00
parent 443f1eecbc
commit 23a201cc84
11 changed files with 431 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

15
breezewiki.rkt
View file

@ -1,5 +1,7 @@
#lang racket/base
(require web-server/servlet-dispatch
(require racket/splicing
web-server/servlet-dispatch
web-server/safety-limits
"src/config.rkt"
"src/dispatcher-tree.rkt"
"src/reloadable.rkt")
@ -9,6 +11,9 @@
(reloadable-entry-point->procedure
(make-reloadable-entry-point (quote varname) filename))))
(require-reloadable "src/page-captcha.rkt" page-captcha)
(require-reloadable "src/page-captcha.rkt" page-captcha-image)
(require-reloadable "src/page-captcha.rkt" page-captcha-verify)
(require-reloadable "src/page-category.rkt" page-category)
(require-reloadable "src/page-global-search.rkt" page-global-search)
(require-reloadable "src/page-home.rkt" page-home)
@ -22,7 +27,9 @@
(require-reloadable "src/page-static-archive.rkt" page-static-archive)
(require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher)
(require-reloadable "src/page-wiki.rkt" page-wiki)
(require-reloadable "src/page-wiki.rkt" page-wiki-with-data)
(require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline)
(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp)
(require-reloadable "src/page-file.rkt" page-file)
(reload!)
@ -34,10 +41,14 @@
(if (config-true? 'debug) "127.0.0.1" #f)
(config-get 'bind_host))
#:port (string->number (config-get 'port))
#:safety-limits (make-safety-limits #:max-request-body-length (* 8 1024 1024))
(λ (quit)
(channel-put ch (lambda () (semaphore-post quit)))
(dispatcher-tree
; order of these does not matter
page-captcha
page-captcha-image
page-captcha-verify
page-category
page-global-search
page-home
@ -48,7 +59,9 @@
page-set-user-settings
page-static-archive
page-wiki
page-wiki-with-data
page-wiki-offline
page-wiki-jsonp
page-file
redirect-wiki-home
static-dispatcher

10
lib/tree-updater.rkt
View file

@ -58,6 +58,16 @@
(data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png")
(class "thumbimage")))))
(figcaption "Test figure!"))
(div (@ (type "slideshow") (position "center") (widths "500") (mode "slideshow") (seq-no "0") (id "slideshow-0") (hash "b62d0efee427ad7dff1026e6e9dd078c") (class "wikia-slideshow wikia-gallery slideshow-center"))
(div (@ (class "wikia-slideshow-wrapper") (style "width: 500px") (data-test-outer-width))
(div (@ (class "wikia-slideshow-images-wrapper"))
(ul (@ (class "wikia-slideshow-images neutral") (style "height: 375px; width: 500px") (data-test-inner-width))
(li (@ (class "wikia-slideshow-first-image"))
(a (@ (class "image lightbox") (title "Expand slideshow") (id "slideshow-0-0") (style "width: 420px"))
(img (@ (data-src "https://static.wikia.nocookie.net/example/images/3/3d/Image.jpg/revision/latest/scale-to-width-down/500?cb=20140129105112") (class "thumbimage") (width "500") (height "281") (style "border: 0px;"))))
(div (@ (class "wikia-slideshow-overlay"))
(div (@ (class "wikia-slideshow-image-caption"))
"Example caption")))))))
(iframe (@ (src "https://example.com/iframe-src")))
(div (@ (class "reviews"))
(header "GameSpot Expert Reviews"))

6
src/config.rkt
View file

@ -49,6 +49,12 @@
(feature_offline::only . "false")
(feature_offline::search . "fandom")
(feature_jsonp::enabled . "true")
(captcha::enabled . "false")
(captcha::log . "false")
(captcha::ip_header . "")
(access_log::enabled . "false")
(promotions::indie_wiki_buddy . "banner home")))

12
src/data.rkt
View file

@ -20,6 +20,7 @@
(struct-out license^)
(struct-out head-data^)
(struct-out user-cookies^)
data->siteinfo
siteinfo-fetch
siteinfo-default
license-default
@ -66,12 +67,15 @@
("formatversion" . "2"))))
(cond [(= (easy:response-status-code res) 200)
(define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))]
(data->siteinfo data)]
[else siteinfo-default])]))
(define (data->siteinfo data)
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data))))
(define/memoize (head-data-getter wikiname) #:hash hash
;; data will be stored here, can be referenced by the memoized closure
(define this-data head-data-default)

10
src/dispatcher-tree.rkt
View file

@ -56,15 +56,25 @@
(sequencer:make
subdomain-dispatcher
(pathprocedure:make "/" (page ds page-home))
(filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher))
(filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image)))
(filter:make (pregexp "^/captcha/verify/[0-9]+/[0-9]+/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-verify)))
(if (config-true? 'captcha::enabled)
(lift:make (page ds page-captcha))
(λ (_conn _req) (next-dispatcher)))
(pathprocedure:make "/proxy" (page ds page-proxy))
(pathprocedure:make "/search" (page ds page-global-search))
(pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
(pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
(if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
(λ (_conn _req) (next-dispatcher)))
(if (config-true? 'feature_jsonp::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp)))
(λ (_conn _req) (next-dispatcher)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))

177
src/page-captcha.rkt Normal file
View file

@ -0,0 +1,177 @@
#lang racket/base
(require racket/class
racket/dict
racket/draw
pict
file/convertible
racket/format
racket/list
racket/math
racket/match
web-server/http
(only-in web-server/dispatchers/dispatch next-dispatcher)
net/url
(only-in net/cookies/server cookie->set-cookie-header cookie-header->alist)
html-writing
"application-globals.rkt"
"data.rkt"
"config.rkt"
"static-data.rkt"
"../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
page-captcha
page-captcha-image
page-captcha-verify)
(define (get-ip req)
(define header
(if (config-true? 'captcha::ip_header)
(headers-assq* (string->bytes/utf-8 (config-get 'captcha::ip_header)) (request-headers/raw req))
#f))
(if header
(~a (header-value header))
(request-client-ip req)))
(define (get-rng req)
(parameterize ([current-pseudo-random-generator (make-pseudo-random-generator)])
(define ip-segments (regexp-match* "[0-9]+" (get-ip req)))
(define seed
(modulo
(for/sum ([i (in-naturals)]
[s ip-segments])
(* (add1 i) (add1 (string->number s))))
(expt 2 32)))
(random-seed seed)
(current-pseudo-random-generator)))
(define (get-key-solution req)
(parameterize ([current-pseudo-random-generator (get-rng req)])
(random 1 (add1 9))))
(define diameter 35)
(define font (make-object font% 12 'system))
(define msg "I'm not a robot!")
(define checkbox (filled-ellipse diameter diameter #:color "Pale Goldenrod"))
(define assembly
(frame
(inset
(hc-append
8
checkbox
(text msg font))
8)))
(define-values (inner-x inner-y) (cc-find assembly checkbox))
(define-values (lt-x lt-y) (lt-find assembly checkbox))
(define-values (rb-x rb-y) (rb-find assembly checkbox))
(define (get-coordinate-solution req w h)
(parameterize ([current-pseudo-random-generator (get-rng req)])
(values (random (exact-truncate lt-x) (exact-truncate (- w rb-x)))
(random (exact-truncate lt-y) (exact-truncate (- h rb-y))))))
(define (page-captcha req)
(define cookie-header (headers-assq* #"cookie" (request-headers/raw req)))
(define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null))
(for ([pair cookies-alist])
(match pair
[(cons "captcha" method)
(when (config-true? 'captcha::log)
(printf "captcha skip - via ~a [~a] - ~v~n" method (get-ip req) (url->string (request-uri req))))
(next-dispatcher)]
[_ (void)]))
(response-handler
(define body
`(*TOP*
(*DECL* DOCTYPE html)
(html
(head
(meta (@ (name "viewport") (content "width=device-width, initial-scale=1")))
(title "Checking you're not a bot...")
(link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css"))))
(link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css"))))
(link (@ (rel "icon") (href ,(head-data^-icon-url head-data-default))))
(script (@ (defer) (src "/static/captcha.js")))
(body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal"))
(div (@ (class "main-container"))
(div (@ (class "fandom-community-header__background tileBoth header")))
(div (@ (class "page"))
(main (@ (class "page__main"))
(div (@ (class "custom-top"))
(h1 (@ (class "page-title"))
"Checking you're not a bot..."))
(div (@ (id "content") #;(class "page-content"))
(div (@ (id "mw-content-text"))
(p "To confirm, please click directly in the circle, or hold down the " ,(~a (get-key-solution req)) " key on your keyboard.")
(noscript (p "JavaScript is required for the captcha. Sorry!"))
(div (@ (id "captcha-area")))))
,(application-footer #f)))))))))
(when (config-true? 'debug)
(xexp->html body))
(response/output
#:code 200
#:headers always-headers
(λ (out)
(write-html body out)))))
(define (page-captcha-image req)
(response-handler
(define w (string->number (path/param-path (third (url-path (request-uri req))))))
(define h (string->number (path/param-path (fourth (url-path (request-uri req))))))
(define-values (at-x at-y) (get-coordinate-solution req w h))
(when (config-true? 'captcha::log)
(printf "captcha show - size ~a x ~a - solution ~a x ~a [~a]~n" w h at-x at-y (get-ip req)))
#;(printf "target: ~a x ~a~ncanvas: ~a x ~a~npict size: ~a-~a ~a-~a~n" at-x at-y x y lt-x rb-x lt-y rb-y)
(define dc (make-object bitmap-dc% #f))
(send dc set-font font)
(define bm (make-object bitmap% w h #f #f))
(send dc set-bitmap bm)
(draw-pict
assembly
dc
(- at-x inner-x)
(- at-y inner-y))
(define image (convert bm 'png-bytes))
(response/output
#:mime-type #"image/png"
#:headers (list (header #"Cache-Control" #"no-cache"))
(λ (out) (display image out)))))
(define (page-captcha-verify req)
(response-handler
(match-define (list w h x y) (for/list ([segment (drop (url-path (request-uri req)) 2)])
(string->number (path/param-path segment))))
#;(printf "solution: ~a x ~a~ncoordinate: ~a x ~a~ndist^2: ~a~n" solution-x solution-y x y dist)
(define headers
(build-headers
always-headers
(cond
[(and (= y 0) (= x (get-key-solution req)))
(when (config-true? 'captcha::log)
(printf "captcha pass - key ~a [~a]~n" x (get-ip req)))
(header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10))))]
[else
(when (config-true? 'captcha::log)
(printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))])
(when (> y 0)
(let-values ([(solution-x solution-y) (get-coordinate-solution req w h)])
(let ([dist (+ (expt (- x solution-x) 2) (expt (- y solution-y) 2))])
(cond
[(dist . < . (expt (/ diameter 2) 2))
(when (config-true? 'captcha::log)
(printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req)))
(header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "coordinate" #:path "/" #:max-age (* 60 60 24 365 10))))]
[else
(when (config-true? 'captcha::log)
(printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req)))]))))))
(match (dict-ref (url-query (request-uri req)) 'from #f)
[(? string? dest)
(response/output
#:code 302
#:mime-type #"text/plain"
#:headers (cons (header #"Location" (string->bytes/utf-8 dest)) headers)
(λ (out)
(displayln "Checking your answer..." out)))]
[#f (next-dispatcher)])))

68
src/page-wiki-jsonp.rkt Normal file
View file

@ -0,0 +1,68 @@
#lang racket/base
(require racket/list
racket/string
web-server/http
net/url-structs
html-writing
"application-globals.rkt"
"data.rkt"
"config.rkt"
"../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt"
"../lib/archive-file-mappings.rkt")
(provide
page-wiki-jsonp)
(define (page-wiki-jsonp req)
(response-handler
(define wikiname (path/param-path (first (url-path (request-uri req)))))
(define segments (map path/param-path (cdr (url-path (request-uri req)))))
(define path (string-join (cdr segments) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define wiki-page-script-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query `(("action" . "parse")
("page" . ,path)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")
("callback" . "wikiPageCallback")))))
(define siteinfo-script-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query `(("action" . "query")
("meta" . "siteinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2")
("callback" . "siteinfoCallback")))))
(define body
(generate-wiki-page
`(div
(noscript "You have to enable JavaScript to load wiki pages. Sorry!")
(div (@ (id "loading")))
(progress (@ (id "progress") (style "margin-bottom: 50vh")))
(script ,(format #<<END
var wikiname = ~v;
var path = ~v;
END
wikiname path))
(script (@ (src "/static/jsonp.js")))
(script (@ (async) (src ,wiki-page-script-url)))
(script (@ (async) (src ,siteinfo-script-url))))
#:req req
#:source-url source-url
#:wikiname wikiname
#:title (url-segments->guess-title segments)
#:siteinfo siteinfo-default))
(when (config-true? 'debug)
(xexp->html body))
(response/output
#:code 200
#:headers always-headers
(λ (out)
(write-html body out)))))

98
src/page-wiki.rkt
View file

@ -6,6 +6,7 @@
racket/string
; libs
(prefix-in easy: net/http-easy)
json
; html libs
"../lib/html-parsing/main.rkt"
html-writing
@ -19,8 +20,6 @@
"data.rkt"
"fandom-request.rkt"
"../lib/archive-file-mappings.rkt"
"../lib/pure-utils.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/tree-updater.rkt"
"../lib/url-utils.rkt"
@ -29,6 +28,7 @@
(provide
; used by the web server
page-wiki
page-wiki-with-data
; used by page-category, and similar pages that are partially wiki pages
update-tree-wiki
preprocess-html-wiki)
@ -59,48 +59,10 @@
(cond
[(eq? 200 (easy:response-status-code dest-res))
(let* ([data (easy:response-json dest-res)]
[title (jp "/parse/title" data "")]
[page-html (jp "/parse/text" data "")]
[page-html (preprocess-html-wiki page-html)]
[page (html->xexp page-html)]
[head-data ((head-data-getter wikiname) data)])
(let ([data (easy:response-json dest-res)])
(if (equal? "missingtitle" (jp "/error/code" data #f))
(next-dispatcher)
(response-handler
(define body
(generate-wiki-page
(update-tree-wiki page wikiname)
#:req req
#:source-url source-url
#:wikiname wikiname
#:title title
#:head-data head-data
#:siteinfo siteinfo))
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
(define redirect-msg-a (if redirect-msg
((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
#f))
(define headers
(build-headers
always-headers
; redirect-query-parameter: only the string "no" is significant:
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
(when (and redirect-msg-a
(not (equal? redirect-query-parameter "no")))
(let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
(header #"Refresh" value)))))
(when (config-true? 'debug)
; used for its side effects
; convert to string with error checking, error will be raised if xexp is invalid
(xexp->html body))
(response/output
#:code 200
#:headers headers
(λ (out)
(write-html body out))))))]
(take-json-rewrite-and-return-page data)))]
[(eq? 404 (easy:response-status-code dest-res))
(next-dispatcher)]
[(memq (easy:response-status-code dest-res) '(403 406))
@ -132,3 +94,55 @@
path
(easy:response-status-code dest-res)
(easy:response-body dest-res)))]))
(define (page-wiki-with-data req)
(define post-data/bytes (request-post-data/raw req))
(define post-data/string (bytes->string/utf-8 post-data/bytes))
(define post-data (string->jsexpr post-data/string))
(define wikiname (jp "/wikiname" post-data))
(define path (jp "/path" post-data))
(take-json-rewrite-and-return-page
#:req req
#:wikiname wikiname
#:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
#:data (jp "/data" post-data)
#:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))
(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo)
(define title (jp "/parse/title" data ""))
(define page-html (preprocess-html-wiki (jp "/parse/text" data "")))
(define page (html->xexp page-html))
(define head-data ((head-data-getter wikiname) data))
(response-handler
(define body
(generate-wiki-page
(update-tree-wiki page wikiname)
#:req req
#:source-url source-url
#:wikiname wikiname
#:title title
#:head-data head-data
#:siteinfo siteinfo))
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
(define redirect-msg-a (if redirect-msg
((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
#f))
(define html (xexp->html-bytes body))
(define headers
(build-headers
always-headers
; redirect-query-parameter: only the string "no" is significant:
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
(when (and redirect-msg-a
(not (equal? redirect-query-parameter "no")))
(let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
(header #"Refresh" value)))))
(response/full
200
#"OK"
(current-seconds)
#"text/html; charset=utf-8"
headers
(list html))))

24
static/captcha.js Normal file
View file

@ -0,0 +1,24 @@
const u = new URL(location)
const from = u.searchParams.get("from") || location.href
let answered = false
const area = document.getElementById("captcha-area")
const areaBox = area.getBoundingClientRect()
const width = Math.floor(areaBox.width)
const height = Math.floor(window.innerHeight - areaBox.bottom - areaBox.left)
const img = document.createElement("img")
img.src = `/captcha/img/${width}/${height}`
img.addEventListener("click", event => {
if (answered) return
answered = true
location = `/captcha/verify/${width}/${height}/${event.offsetX}/${event.offsetY}?` + new URLSearchParams({from})
})
area.appendChild(img)
document.addEventListener("keydown", event => {
if (event.repeat) {
if (answered) return
answered = true
location = `/captcha/verify/0/0/${event.key}/0?` + new URLSearchParams({from})
}
})

57
static/jsonp.js Normal file
View file

@ -0,0 +1,57 @@
const loading = document.getElementById("loading")
loading.textContent = "Loading, please wait..."
const progress = document.getElementById("progress")
let wikiPage = null
function wikiPageCallback(data) {
wikiPage = data
cont()
}
let siteinfo = null
function siteinfoCallback(data) {
siteinfo = data
cont()
}
async function cont() {
if (!(wikiPage && siteinfo)) return
const xhr = new XMLHttpRequest();
const uploadFraction = 0.7
// Upload progress
xhr.upload.addEventListener("progress", event => {
if (event.lengthComputable) {
progress.value = (event.loaded / event.total) * uploadFraction
console.log(
`Uploaded ${((event.loaded / event.total) * 100).toFixed(2)}%`,
)
}
})
// Download progress
xhr.addEventListener("progress", event => {
if (event.lengthComputable) {
progress.value = (event.loaded / event.total) * (1 - uploadFraction) + uploadFraction
console.log(
`Downloaded ${((event.loaded / event.total) * 100).toFixed(2)}%`,
)
}
})
xhr.addEventListener("load", () => {
console.log(xhr)
document.body = xhr.responseXML.body
})
xhr.open("POST", "/api/render/wiki")
xhr.responseType = "document"
xhr.send(JSON.stringify({
data: wikiPage,
siteinfo,
wikiname,
path
}));
}

1
static/robots.txt
View file

@ -2,3 +2,4 @@ User-Agent: *
Disallow: /*/wiki/*
Disallow: /proxy
Disallow: /set-user-settings
Disallow: /captcha