fix cache control for jsonp.js

JSONP mode is on by default. It will fetch main wiki pages in the
browser, without the server needing to make any requests. To turn it
off, add [feature_json] enabled = false to config.ini.

Captcha is off by default. It is a custom solution and is still
experimental at this stage. If you turn it on, please monitor the logs
to see how it goes! config.ini options are as follows:

[captcha]
enabled = true|false
log = true|false
ip_header = <header name set by your reverse proxy, like x-forwarded-for>

archiver/fts.rkt

@@ -8,6 +8,7 @@
          racket/promise
          racket/port
          racket/runtime-path
+         racket/sequence
          racket/string
          file/gunzip
          db
@@ -156,8 +157,16 @@
  (define data
    (cond
      [(and (read-from-cache?) (file-exists? "cache.rkt"))
-        (displayln "Reading in...")
-        (with-input-from-file "cache.rkt" (λ () (read)))]
+      (define size (file-size "cache.rkt"))
+      (call-with-input-file "cache.rkt"
+        (λ (in)
+          (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position in) 64 1024))
+                                                       (ceiling (/ size 64 1024))
+                                                       "Reading in..."))
+                                      2))
+          (begin0
+              (read in)
+            (quit))))]
      [else
       (define x (box (progress^ 0 1 "...")))
       (define quit (make-progress (λ () (unbox x))))
@@ -183,18 +192,22 @@
 
  (display "Converting... ")
  (flush-output)
- (define ser (jsexpr->bytes data))
- (define ser-port (open-input-bytes ser))
- (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024))
-                                              (ceiling (/ (bytes-length ser) 64 1024))
-                                              "Posting..."))
-                             2))
- (define res
-   (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname)
-         #:data ser-port
-         #:headers '#hasheq((Content-Type . "application/json"))
-         #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300)))
- (quit)
- (displayln (response-status-line res)))
+ (define slice-size 30000)
+ (define slices (ceiling (/ (length data) slice-size)))
+ (for ([slice (in-slice slice-size data)]
+       [i (in-naturals 1)])
+   (define ser (jsexpr->bytes slice))
+   (define ser-port (open-input-bytes ser))
+   (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024))
+                                                (ceiling (/ (bytes-length ser) 64 1024))
+                                                (format "Posting... (~a/~a)" i slices)))
+                               2))
+   (define res
+     (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname)
+           #:data ser-port
+           #:headers '#hasheq((Content-Type . "application/json"))
+           #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300)))
+   (quit)
+ (displayln (response-status-line res))))
 
 (run start)

breezewiki.rkt

@@ -1,5 +1,7 @@
 #lang racket/base
-(require web-server/servlet-dispatch
+(require racket/splicing
+         web-server/servlet-dispatch
+         web-server/safety-limits
          "src/config.rkt"
          "src/dispatcher-tree.rkt"
          "src/reloadable.rkt")
@@ -9,6 +11,9 @@
     (reloadable-entry-point->procedure
      (make-reloadable-entry-point (quote varname) filename))))
 
+(require-reloadable "src/page-captcha.rkt" page-captcha)
+(require-reloadable "src/page-captcha.rkt" page-captcha-image)
+(require-reloadable "src/page-captcha.rkt" page-captcha-verify)
 (require-reloadable "src/page-category.rkt" page-category)
 (require-reloadable "src/page-global-search.rkt" page-global-search)
 (require-reloadable "src/page-home.rkt" page-home)
@@ -22,7 +27,9 @@
 (require-reloadable "src/page-static-archive.rkt" page-static-archive)
 (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher)
 (require-reloadable "src/page-wiki.rkt" page-wiki)
+(require-reloadable "src/page-wiki.rkt" page-wiki-with-data)
 (require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline)
+(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp)
 (require-reloadable "src/page-file.rkt" page-file)
 
 (reload!)
@@ -34,10 +41,14 @@
                  (if (config-true? 'debug) "127.0.0.1" #f)
                  (config-get 'bind_host))
    #:port (string->number (config-get 'port))
+   #:safety-limits (make-safety-limits #:max-request-body-length (* 8 1024 1024))
    (λ (quit)
      (channel-put ch (lambda () (semaphore-post quit)))
      (dispatcher-tree
       ; order of these does not matter
+      page-captcha
+      page-captcha-image
+      page-captcha-verify
       page-category
       page-global-search
       page-home
@@ -48,7 +59,9 @@
       page-set-user-settings
       page-static-archive
       page-wiki
+      page-wiki-with-data
       page-wiki-offline
+      page-wiki-jsonp
       page-file
       redirect-wiki-home
       static-dispatcher

dist.rkt

@@ -3,6 +3,7 @@
          "src/config.rkt"
          "src/dispatcher-tree.rkt")
 
+(require (only-in "src/page-captcha.rkt" page-captcha page-captcha-image page-captcha-verify))
 (require (only-in "src/page-category.rkt" page-category))
 (require (only-in "src/page-global-search.rkt" page-global-search))
 (require (only-in "src/page-home.rkt" page-home))
@@ -15,8 +16,9 @@
 (require (only-in "src/page-static.rkt" static-dispatcher))
 (require (only-in "src/page-static-archive.rkt" page-static-archive))
 (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher))
-(require (only-in "src/page-wiki.rkt" page-wiki))
+(require (only-in "src/page-wiki.rkt" page-wiki page-wiki-with-data))
 (require (only-in "src/page-wiki-offline.rkt" page-wiki-offline))
+(require (only-in "src/page-wiki-jsonp.rkt" page-wiki-jsonp))
 (require (only-in "src/page-file.rkt" page-file))
 
 (serve/launch/wait
@@ -27,6 +29,9 @@
  (λ (quit)
    (dispatcher-tree
     ; order of these does not matter
+    page-captcha
+    page-captcha-image
+    page-captcha-verify
     page-category
     page-global-search
     page-home
@@ -38,6 +43,8 @@
     page-static-archive
     page-wiki
     page-wiki-offline
+    page-wiki-with-data
+    page-wiki-jsonp
     page-file
     redirect-wiki-home
     static-dispatcher

lib/thread-utils.rkt

@@ -1,6 +1,5 @@
 #lang racket/base
-(require (prefix-in easy: net/http-easy)
-         "../src/data.rkt"
+(require "../src/data.rkt"
          "xexpr-utils.rkt")
 
 (provide

lib/tree-updater.rkt

@@ -58,6 +58,16 @@
                                (data-src "https://static.wikia.nocookie.net/nice-image-thumbnail.png")
                                (class "thumbimage")))))
                    (figcaption "Test figure!"))
+           (div (@ (type "slideshow") (position "center") (widths "500") (mode "slideshow") (seq-no "0") (id "slideshow-0") (hash "b62d0efee427ad7dff1026e6e9dd078c") (class "wikia-slideshow wikia-gallery slideshow-center"))
+                (div (@ (class "wikia-slideshow-wrapper") (style "width: 500px") (data-test-outer-width))
+                     (div (@ (class "wikia-slideshow-images-wrapper"))
+                          (ul (@ (class "wikia-slideshow-images neutral") (style "height: 375px; width: 500px") (data-test-inner-width))
+                              (li (@ (class "wikia-slideshow-first-image"))
+                                  (a (@ (class "image lightbox") (title "Expand slideshow") (id "slideshow-0-0") (style "width: 420px"))
+                                     (img (@ (data-src "https://static.wikia.nocookie.net/example/images/3/3d/Image.jpg/revision/latest/scale-to-width-down/500?cb=20140129105112") (class "thumbimage") (width "500") (height "281") (style "border: 0px;"))))
+                                  (div (@ (class "wikia-slideshow-overlay"))
+                                       (div (@ (class "wikia-slideshow-image-caption"))
+                                            "Example caption")))))))
            (iframe (@ (src "https://example.com/iframe-src")))
            (div (@ (class "reviews"))
                 (header "GameSpot Expert Reviews"))

src/application-globals.rkt

@@ -22,8 +22,6 @@
 (provide
  ; headers to always send on all http responses
  always-headers
- ; timeout durations for http-easy requests
- timeouts
  ; generates a consistent footer
  application-footer
  ; generates a consistent template for wiki page content to sit in
@@ -39,7 +37,6 @@
 (define always-headers
   (list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom
         (header #"Link" (string->bytes/latin-1 link-header))))
-(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
 
 (define-runtime-path path-static "../static")
 (define theme-icons
@@ -71,9 +68,6 @@
                           `(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name))
                               (a (@ (href "https://cadence.moe/contact"))
                                  "Cadence")
-                              ". Proudly hosted by "
-                              (a (@ (href "http://alphamethyl.barr0w.net"))
-                                 "Barrow Network Solutions" (sup "XD"))
                               ".")
                           `(p
                             ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name)))))
@@ -91,11 +85,13 @@
 
 ;; generate a notice with a link if a fandom wiki has a replacement as part of NIWA or similar
 ;; if the wiki has no replacement, display nothing
-(define (extwiki-notice wikiname title)
+(define (extwiki-notice wikiname title req user-cookies)
   (define xt (findf (λ (item) (member wikiname (extwiki^-wikinames item))) extwikis))
   (cond/var
    [xt
-    (let* ([group (hash-ref extwiki-groups (extwiki^-group xt))]
+    (let* ([seen? (member wikiname (user-cookies^-notices user-cookies))]
+           [aside-class (if seen? "niwa__notice niwa--seen" "niwa__notice")]
+           [group (hash-ref extwiki-groups (extwiki^-group xt))]
            [search-page (format "/Special:Search?~a"
                                 (params->query `(("search" . ,title)
                                                  ("go" . "Go"))))]
@@ -106,21 +102,24 @@
            [props (extwiki-props^ go)])
       (cond
         [(eq? (extwiki^-banner xt) 'default)
-         `(aside (@ (class "niwa__notice"))
+         `(aside (@ (class ,aside-class))
                  (h1 (@ (class "niwa__header")) ,(extwiki^-name xt) " has its own website separate from Fandom.")
                  (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,(extwiki^-name xt) " →")
                  (div (@ (class "niwa__cols"))
                       (div (@ (class "niwa__left"))
-                           (p ,((extwiki-group^-description group) props))
                            (p ,((extwiki^-description xt) props))
-                           (p "This wiki's core community has wholly migrated away from Fandom. You should "
+                           (p ,((extwiki-group^-description group) props))
+                           (p "This wiki's core community has largely migrated away from Fandom. You should "
                               (a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!"))
                            (p (@ (class "niwa__feedback"))
                               ,@(add-between
                                  `(,@(for/list ([link (extwiki-group^-links group)])
                                        `(a (@ (href ,(cdr link))) ,(car link)))
                                    "This notice is from BreezeWiki"
-                                   (a (@ (href "https://docs.breezewiki.com/Reporting_Bugs.html")) "Feedback?"))
+                                   (a (@ (rel "nofollow")
+                                         (class "niwa__got-it")
+                                         (href ,(user-cookies-setter-url/add-notice req user-cookies wikiname)))
+                                      "OK, got it"))
                                  " / ")))
                       (div (@ (class "niwa__right"))
                            (img (@ (class "niwa__logo") (src ,(extwiki^-logo xt)))))))]
@@ -175,7 +174,7 @@
        (define styles
          (list
           (format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies))
-          (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin)))
+          (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.fandom.photoGallery.gallery.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin)))
        (if (config-true? 'strict_proxy)
            (map u-proxy-url styles)
            styles)]
@@ -203,10 +202,11 @@
            `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js"))))
            "")
       (script (@ (type "module") (src ,(get-static-url "countdown.js"))))
+      (script (@ (defer) (src ,(get-static-url "tabs.js"))))
       (link (@ (rel "icon") (href ,(u (λ (v) (config-true? 'strict_proxy))
                                       (λ (v) (u-proxy-url v))
                                       (head-data^-icon-url head-data))))))
-     (body (@ (class ,(head-data^-body-class head-data)))
+     (body (@ (class ,(head-data^-body-class head-data) " bw-tabs-nojs"))
            ,(let ([extension-eligible?
                    (cond/var
                     [(not req) #f]
@@ -227,7 +227,7 @@
                 (div (@ (class "fandom-community-header__background tileHorizontally header")))
                 (div (@ (class "page"))
                      (main (@ (class "page__main"))
-                           ,(extwiki-notice wikiname title)
+                           ,(extwiki-notice wikiname title req user-cookies)
                            (div (@ (class "custom-top"))
                                 (h1 (@ (class "page-title")) ,title)
                                 (nav (@ (class "sitesearch"))

src/config.rkt

@@ -49,6 +49,12 @@
     (feature_offline::only . "false")
     (feature_offline::search . "fandom")
 
+    (feature_jsonp::enabled . "true")
+
+    (captcha::enabled . "false")
+    (captcha::log . "false")
+    (captcha::ip_header . "")
+
     (access_log::enabled . "false")
 
     (promotions::indie_wiki_buddy . "banner home")))
@@ -104,7 +110,7 @@
   ; all values here are optimised for maximum prettiness
   (parameterize ([pretty-print-columns 80])
     (display "config: ")
-    (pretty-write ((inst sort (Pairof Symbol String))
+    (pretty-write ((inst sort (Pairof Symbol String) Symbol)
                    (hash->list (make-immutable-hasheq combined-alist))
                    symbol<?
                    #:key car))))

src/data.rkt

@@ -1,14 +1,15 @@
 #lang racket/base
 (require racket/list
          racket/match
+         racket/string
          web-server/http/request-structs
          net/url-string
          (only-in net/cookies/server cookie-header->alist cookie->set-cookie-header make-cookie)
          (prefix-in easy: net/http-easy)
          db
          memo
+         "fandom-request.rkt"
          "static-data.rkt"
-         "whole-utils.rkt"
          "../lib/url-utils.rkt"
          "../lib/xexpr-utils.rkt"
          "../archiver/archiver-database.rkt"
@@ -19,6 +20,7 @@
  (struct-out license^)
  (struct-out head-data^)
  (struct-out user-cookies^)
+ data->siteinfo
  siteinfo-fetch
  siteinfo-default
  license-default
@@ -27,7 +29,8 @@
  user-cookies-getter
  user-cookies-default
  user-cookies-setter
- user-cookies-setter-url)
+ user-cookies-setter-url
+ user-cookies-setter-url/add-notice)
 
 (struct siteinfo^ (sitename basepage license) #:transparent)
 (struct license^ (text url) #:transparent)
@@ -54,21 +57,24 @@
                               (vector-ref row 3)))
          siteinfo-default)]
     [else
-     (define dest-url
-       (format "https://~a.fandom.com/api.php?~a"
-               wikiname
-               (params->query '(("action" . "query")
-                                ("meta" . "siteinfo")
-                                ("siprop" . "general|rightsinfo")
-                                ("format" . "json")
-                                ("formatversion" . "2")))))
-     (log-outgoing dest-url)
-     (define res (easy:get dest-url))
-     (define data (easy:response-json res))
-     (siteinfo^ (jp "/query/general/sitename" data)
-                (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
-                (license^ (jp "/query/rightsinfo/text" data)
-                          (jp "/query/rightsinfo/url" data)))]))
+     (define res
+       (fandom-get-api
+        wikiname
+        '(("action" . "query")
+          ("meta" . "siteinfo")
+          ("siprop" . "general|rightsinfo")
+          ("format" . "json")
+          ("formatversion" . "2"))))
+     (cond [(= (easy:response-status-code res) 200)
+            (define data (easy:response-json res))
+            (data->siteinfo data)]
+           [else siteinfo-default])]))
+
+(define (data->siteinfo data)
+  (siteinfo^ (jp "/query/general/sitename" data)
+             (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
+             (license^ (jp "/query/rightsinfo/text" data)
+                       (jp "/query/rightsinfo/url" data))))
 
 (define/memoize (head-data-getter wikiname) #:hash hash
   ;; data will be stored here, can be referenced by the memoized closure
@@ -90,8 +96,8 @@
     ;; then no matter what, return the best information we have so far
     this-data))
 
-(struct user-cookies^ (theme) #:prefab)
-(define user-cookies-default (user-cookies^ 'default))
+(struct user-cookies^ (theme notices) #:prefab)
+(define user-cookies-default (user-cookies^ 'default '()))
 (define (user-cookies-getter req)
   (define cookie-header (headers-assq* #"cookie" (request-headers/raw req)))
   (define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null))
@@ -100,16 +106,29 @@
       (match pair
         [(cons "theme" (and theme (or "light" "dark" "default")))
          (values 'theme (string->symbol theme))]
+        [(cons "notices" notices)
+         (values 'notices (string-split notices "|"))]
         [_ (values #f #f)])))
   (user-cookies^
-   (hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default))))
+   (hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default))
+   (hash-ref cookies-hash 'notices (user-cookies^-notices user-cookies-default))))
 
 (define (user-cookies-setter user-cookies)
   (map (λ (c) (header #"Set-Cookie" (cookie->set-cookie-header c)))
        (list (make-cookie "theme" (symbol->string (user-cookies^-theme user-cookies))
+                          #:path "/"
+                          #:max-age (* 60 60 24 365 10))
+             (make-cookie "notices" (string-join (user-cookies^-notices user-cookies) "|")
+
                           #:path "/"
                           #:max-age (* 60 60 24 365 10)))))
 
 (define (user-cookies-setter-url req new-settings)
   (format "/set-user-settings?~a"  (params->query `(("next_location" . ,(url->string (request-uri req)))
-                                                    ("new_settings" . ,(format "~a" new-settings))))))
+                                                    ("new_settings" . ,(format "~s" new-settings))))))
+
+(define (user-cookies-setter-url/add-notice req user-cookies notice-name)
+  (user-cookies-setter-url
+   req
+   (struct-copy user-cookies^ user-cookies
+                [notices (cons notice-name (user-cookies^-notices user-cookies))])))

src/dispatcher-tree.rkt

@@ -33,26 +33,53 @@
   ; don't forget that I'm returning *code* - return a call to the function
   (datum->syntax stx `(make-dispatcher-tree ,ds)))
 
+; guard that the page returned a response, otherwise print more detailed debugging information
+(define-syntax-rule (page ds name)
+  (λ (req)
+    (define dispatcher (hash-ref ds (quote name)))
+    (define page-response (dispatcher req))
+    (if (response? page-response)
+        page-response
+        (response/output
+         #:code 500
+         #:mime-type #"text/plain"
+         (λ (out)
+           (for ([port (list (current-error-port) out)])
+             (parameterize ([current-output-port port])
+               (printf "error in ~a:~n  expected page to return a response~n  actually returned: ~v~n"
+                       (quote name)
+                       page-response))))))))
+
 (define (make-dispatcher-tree ds)
   (define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher))
   (define tree
     (sequencer:make
      subdomain-dispatcher
-     (pathprocedure:make "/" (hash-ref ds 'page-home))
-     (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy))
-     (pathprocedure:make "/search" (hash-ref ds 'page-global-search))
-     (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings))
-     (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works))
-     (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category)))
-     (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file)))
-     (if (config-true? 'feature_offline::enabled)
-         (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline)))
+     (pathprocedure:make "/" (page ds page-home))
+     (filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher))
+     (filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image)))
+     (filter:make (pregexp "^/captcha/verify/[0-9]+/[0-9]+/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-verify)))
+     (if (config-true? 'captcha::enabled)
+         (lift:make (page ds page-captcha))
          (λ (_conn _req) (next-dispatcher)))
-     (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki)))
-     (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search)))
-     (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home)))
+     (pathprocedure:make "/proxy" (page ds page-proxy))
+     (pathprocedure:make "/search" (page ds page-global-search))
+     (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
+     (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
+     (pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data))
+     (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
+     (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
      (if (config-true? 'feature_offline::enabled)
-         (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive)))
+         (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
+         (λ (_conn _req) (next-dispatcher)))
+     (if (config-true? 'feature_jsonp::enabled)
+         (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp)))
+         (λ (_conn _req) (next-dispatcher)))
+     (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
+     (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
+     (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))
+     (if (config-true? 'feature_offline::enabled)
+         (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive)))
          (λ (_conn _req) (next-dispatcher)))
      (hash-ref ds 'static-dispatcher)
      (lift:make (hash-ref ds 'page-not-found))))

src/extwiki-data.rkt

@@ -27,6 +27,13 @@
            (λ (props)
              `(p "The Square Enix Indpendent Wiki Alliance, or SEIWA, is a network of independent wikis established in 2011 and focused on providing high-quality coverage of Square Enix and its content. We work together, along with our affiliates and others, to co-operate and support one another while providing the best-quality content on the various Square Enix video games and media.")))
 
+          'GWN
+          (extwiki-group^
+           "GWN"
+           '(("Gaming Wiki Network" . "https://gamingwikinetwork.org/"))
+           (λ (props)
+             `(p "This wiki is part of the Gaming Wiki Network, a network of independently-hosted wikis about video game franchises. The GWN was founded on October 21, 2022. It aims to support all gaming communities in building independently-hosted wikis.")))
+
           'Terraria
           (extwiki-group^
            "Terraria"
@@ -71,11 +78,25 @@
            (λ (props)
              '(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop.")))
 
+          'Tardis
+          (extwiki-group^
+           "Tardis"
+           '(("Forking announcement" . "https://tardis.wiki/wiki/Tardis:Forking_announcement")
+             ("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/"))
+           (λ (props) '()))
+
+          'Rainverse
+          (extwiki-group^
+           "Rainverse"
+           '(("Forking announcement" . "https://transfem.social/notes/9qsqdkmqi78e01bh"))
+           (λ (props)
+             '()))
+
           'empty
           (extwiki-group^
            "Misc"
            '(("This wiki doesn't have a description yet. Add one?" . "https://docs.breezewiki.com/Reporting_Bugs.html"))
-           #f)))
+           (λ (props) '()))))
 
 ;; wikiname, niwa-name, url, logo-url
 (struct extwiki^ (wikinames banner group name home logo description) #:transparent)
@@ -309,11 +330,11 @@
    (extwiki^
     '("zelda" "zelda-archive") 'default
     'NIWA
-    "Zeldapedia"
-    "https://zeldapedia.wiki/wiki/Main_Page"
+    "Zelda Wiki"
+    "https://zeldawiki.wiki/wiki/Main_Page"
     "https://niwanetwork.org/images/logos/zeldapedia.png"
     (λ (props)
-      `((p "Founded on April 23, 2005 as Zelda Wiki, today's Zeldapedia is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zeldapedia went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons."))))
+      `((p "Founded on April 23, 2005, Zelda Wiki is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zelda Wiki went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons."))))
 
    (extwiki^
     '("chrono") 'default
@@ -419,6 +440,99 @@
     (λ (props)
       `()))
 
+   (extwiki^
+    '("tardis") 'default
+    'Tardis
+    "TARDIS Wiki"
+    "https://tardis.wiki/wiki/Doctor_Who_Wiki"
+    "https://tardis.wiki/w/images/Tardis_Images/e/e6/Site-logo.png"
+    (λ (props)
+      `()))
+
+   (extwiki^
+    '("wizardry") 'default
+    'GWN
+    "Wizardry Wiki"
+    "https://wizardry.wiki.gg/wiki/Wizardry_Wiki"
+    "https://wizardry.wiki.gg/images/e/e6/Site-logo.png"
+    (λ (props)
+      `((p "On March 21, 2023, the wiki has decided to leave and abandoning from Fandom due to numerous of issues such as intrusive advertising, long-lasting bugs, restrictions on customization, etcetera. Wizardry Wiki was officially inducted into the wiki.gg wikifarm, with all contents forked over.")
+        (p "The wiki has partnered with " (a (@ (href "https://fallout.wiki/")) "Independent Fallout Wiki") " as of June 14, 2024."))))
+
+   (extwiki^
+    '("jackryan") 'default
+    'GWN
+    "Tom Clancy Wiki"
+    "https://tomclancy.wiki.gg/wiki/Tom_Clancy_Wiki"
+    "https://tomclancy.wiki.gg/images/thumb/c/c5/Jack_Ryan_Logo_Dark.png/600px-Jack_Ryan_Logo_Dark.png"
+    (λ (props)
+      `((p "The Tom Clancy Wiki is a collaborative encyclopedia dedicated to Tom Clancy’s franchises. The Tom Clancy franchise is a 40-year old expansive franchise founded by Tom Clancy, telling several unique sagas through books, video games, and films, as well as a TV show."))))
+
+    (extwiki^
+     '("hollowknight") 'default
+     'GWN
+     "Hollow Knight Wiki"
+     "https://hollowknight.wiki/wiki/Main_Page"
+     "https://gamingwikinetwork.org/images/logos/hollowknight.png"
+     (λ (props)
+       `((p "We are an independently hosted wiki for the games Hollow Knight and Hollow Knight: Silksong, created by fans, for fans. The wiki is a fork of the FANDOM Hollow Knight Wiki and was officially unveiled on October 31, 2023."))))
+
+    (extwiki^
+     '("hellokitty" "sanrio") 'default
+     'GWN
+     "Sanrio Wiki"
+     "https://sanriowiki.com/wiki/Sanrio_Wiki"
+     "https://cdn.sanriowiki.com/wiki.png"
+     (λ (props)
+       `((p "Sanrio Wiki is a project that was started on April 14, 2015 by EvieMelody. It was hosted on the wiki-farm ShoutWiki and has since become independent."))))
+
+    (extwiki^
+     '("sto") 'default
+     'GWN
+     "Star Trek Online Wiki"
+     "https://stowiki.net/wiki/Main_Page"
+     "https://gamingwikinetwork.org/images/logos/stowiki.png"
+     (λ (props)
+       `()))
+
+    (extwiki^
+     '("rayman-game" "ubisoftrayman") 'default
+     'GWN
+     "Rayman Wiki"
+     "https://raymanpc.com/wiki/en/Main_Page"
+     "https://raymanpc.com/wiki/script-en/resources/assets/logo-en.png?5c608"
+     (λ (props)
+       `()))
+
+    (extwiki^
+     '("granblue") 'empty
+     'empty
+     "Granblue Fantasy Wiki"
+     "https://gbf.wiki/"
+     "https://gbf.wiki/images/1/18/Vyrnball.png?0704c"
+     (λ (props)
+       `()))
+
+    (extwiki^
+     '("hellmet-roblox") 'empty
+     'empty
+     "HELLMET Wiki"
+     "https://hellmet.miraheze.org/wiki/Main_Page"
+     "https://static.miraheze.org/hellmetwiki/thumb/c/ce/Hellmet_Wiki_Logo.png/135px-Hellmet_Wiki_Logo.png"
+     (λ (props)
+       `()))
+
+    (extwiki^
+     '("rain-web-comic") 'default
+     'empty
+     "Rainverse Wiki"
+     "https://rainverse.wiki/wiki/Main_Page"
+     "https://static.miraheze.org/rainversewiki/2/2c/Rain_comic_cover.png"
+     (λ (props)
+       `((p "We have a newly-migrated Rainverse Wiki which escaped from Fandom! Rain is the comic that helped me figure out my gender, so I am really glad to have a wiki on a non-evil host.")
+         (p "Please stop using the abandoned copy of Rain Wiki on Fandom. Fandom is still \"training\" a generator which adds procedurally-generated bullshit to articles, with no way for users to remove or correct it, and they're demanding volunteer wiki admins waste time \"vetting\" the procedurally-generated BS for accuracy. As Jocelyn herself said, \"fuck Fandom forever.\"")
+         (p "If you are interested, please add more articles related to other Rainverse stories."))))
+
    ;; fandom wikinames * empty * empty * Name * Home Page
    (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f)
    (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f)
@@ -434,6 +548,7 @@
    (extwiki^ '("doom") 'empty 'empty "DoomWiki.org" "https://doomwiki.org/wiki/Entryway" #f #f)
    (extwiki^ '("dreamscaper") 'empty 'empty "Official Dreamscaper Wiki" "https://dreamscaper.wiki.gg/wiki/Dreamscaper_Wiki" #f #f)
    (extwiki^ '("elderscrolls") 'empty 'empty "UESP" "https://en.uesp.net/wiki/Main_Page" #f #f)
+   (extwiki^ '("enterthegungeon" "exit-the-gungeon" "enter-the-gungeon-archive") 'empty 'empty "Official Enter The Gungeon Wiki" "https://enterthegungeon.wiki.gg/wiki/Enter_the_Gungeon_Wiki" "https://enterthegungeon.wiki.gg/images/e/e6/Site-logo.png" #f)
    (extwiki^ '("fiend-folio") 'empty 'empty "Official Fiend Folio Wiki" "https://fiendfolio.wiki.gg/wiki/Fiend_Folio_Wiki" #f #f)
    (extwiki^ '("foxhole") 'empty 'empty "Foxhole Wiki" "https://foxhole.wiki.gg/wiki/Foxhole_Wiki" #f #f)
    (extwiki^ '("have-a-nice-death") 'empty 'empty "Have a Nice Death Wiki" "https://haveanicedeath.wiki.gg/wiki/Have_a_Nice_Death_Wiki" #f #f)

src/fandom-request.rkt

@@ -0,0 +1,74 @@
+#lang typed/racket/base
+(require racket/format
+         racket/string
+         "config.rkt"
+         "../lib/url-utils.rkt")
+(define-type Headers (HashTable Symbol (U Bytes String)))
+(require/typed net/http-easy
+  [#:opaque Timeout-Config timeout-config?]
+  [#:opaque Response response?]
+  [#:opaque Session session?]
+  [response-status-code (Response -> Natural)]
+  [current-session (Parameter Session)]
+  [current-user-agent (Parameter (U Bytes String))]
+  [make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)]
+  [get ((U Bytes String)
+        [#:close? Boolean]
+        [#:headers Headers]
+        [#:timeouts Timeout-Config]
+        [#:max-attempts Exact-Positive-Integer]
+        [#:max-redirects Exact-Nonnegative-Integer]
+        [#:user-agent (U Bytes String)]
+        -> Response)])
+
+(provide
+ fandom-get
+ fandom-get-api
+ timeouts)
+
+(unless (string-contains? (~a (current-user-agent)) "BreezeWiki")
+  (current-user-agent
+   (format "BreezeWiki/1.0 (~a) ~a"
+           (if (config-true? 'canonical_origin)
+               (config-get 'canonical_origin)
+               "local")
+           (current-user-agent))))
+
+(define timeouts (make-timeout-config #:lease 5 #:connect 5))
+
+(: last-failure Flonum)
+(define last-failure 0.0)
+(: stored-failure (Option Response))
+(define stored-failure #f)
+(define failure-persist-time 30000)
+
+(: no-headers Headers)
+(define no-headers '#hasheq())
+
+(: fandom-get (String String [#:headers (Option Headers)] -> Response))
+(define (fandom-get wikiname path #:headers [headers #f])
+  (or
+   (and ((current-inexact-milliseconds) . < . (+ last-failure failure-persist-time)) stored-failure)
+   (let ()
+     (define dest-url (string-append "https://www.fandom.com" path))
+     (define host (string-append wikiname ".fandom.com"))
+     (log-outgoing wikiname path)
+     (define res
+       (get dest-url
+            #:timeouts timeouts
+            #:headers (hash-set (or headers no-headers) 'Host host)))
+     (when (memq (response-status-code res) '(403 406))
+       (set! last-failure (current-inexact-milliseconds))
+       (set! stored-failure res))
+     res)))
+
+(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response))
+(define (fandom-get-api wikiname params #:headers [headers #f])
+  (fandom-get wikiname
+              (string-append "/api.php?" (params->query params))
+              #:headers headers))
+
+(: log-outgoing (String String -> Void))
+(define (log-outgoing wikiname path)
+  (when (config-true? 'log_outgoing)
+    (printf "out: ~a ~a~n" wikiname path)))

src/page-captcha.rkt

@@ -0,0 +1,178 @@
+#lang racket/base
+(require racket/class
+         racket/dict
+         racket/draw
+         pict
+         file/convertible
+         racket/format
+         racket/list
+         racket/math
+         racket/match
+         web-server/http
+         (only-in web-server/dispatchers/dispatch next-dispatcher)
+         net/url
+         (only-in net/cookies/server cookie->set-cookie-header cookie-header->alist)
+         html-writing
+         "application-globals.rkt"
+         "data.rkt"
+         "config.rkt"
+         "static-data.rkt"
+         "../lib/url-utils.rkt"
+         "../lib/xexpr-utils.rkt")
+
+(provide
+ page-captcha
+ page-captcha-image
+ page-captcha-verify)
+
+(define (get-ip req)
+  (define header
+    (if (config-true? 'captcha::ip_header)
+        (headers-assq* (string->bytes/utf-8 (config-get 'captcha::ip_header)) (request-headers/raw req))
+        #f))
+  (if header
+      (~a (header-value header))
+      (request-client-ip req)))
+
+(define (get-rng req)
+  (parameterize ([current-pseudo-random-generator (make-pseudo-random-generator)])
+    (define ip-segments (regexp-match* "[0-9]+" (get-ip req)))
+    (define seed
+      (modulo
+       (for/sum ([i (in-naturals)]
+                 [s ip-segments])
+         (* (add1 i) (add1 (string->number s))))
+       (expt 2 32)))
+    (random-seed seed)
+    (current-pseudo-random-generator)))
+
+(define (get-key-solution req)
+  (parameterize ([current-pseudo-random-generator (get-rng req)])
+    (random 1 (add1 9))))
+
+(define diameter 35)
+(define font (make-object font% 12 'system))
+(define msg "I'm not a robot!")
+(define checkbox (filled-ellipse diameter diameter #:color "Pale Goldenrod"))
+(define assembly
+  (frame
+   (inset
+    (hc-append
+     8
+     checkbox
+     (text msg font))
+    8)))
+(define-values (inner-x inner-y) (cc-find assembly checkbox))
+(define-values (lt-x lt-y) (lt-find assembly checkbox))
+(define-values (rb-x rb-y) (rb-find assembly checkbox))
+
+(define (get-coordinate-solution req w h)
+  (parameterize ([current-pseudo-random-generator (get-rng req)])
+    (values (random (exact-truncate lt-x) (exact-truncate (- w rb-x)))
+            (random (exact-truncate lt-y) (exact-truncate (- h rb-y))))))
+
+(define (page-captcha req)
+  (define cookie-header (headers-assq* #"cookie" (request-headers/raw req)))
+  (define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null))
+  (for ([pair cookies-alist])
+    (match pair
+      [(cons "captcha" method)
+       (when (config-true? 'captcha::log)
+         (printf "captcha skip - via ~a [~a] - ~v~n" method (get-ip req) (url->string (request-uri req))))
+       (next-dispatcher)]
+      [_ (void)]))
+  (response-handler
+   (define body
+     `(*TOP*
+       (*DECL* DOCTYPE html)
+       (html
+        (head
+         (meta (@ (name "viewport") (content "width=device-width, initial-scale=1")))
+         (title "Checking you're not a bot...")
+         (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "internal.css"))))
+         (link (@ (rel "stylesheet") (type "text/css") (href ,(get-static-url "main.css"))))
+         (link (@ (rel "icon") (href ,(head-data^-icon-url head-data-default))))
+         (script (@ (defer) (src "/static/captcha.js")))
+         (body (@ (class "skin-fandomdesktop theme-fandomdesktop-light internal"))
+               (div (@ (class "main-container"))
+                    (div (@ (class "fandom-community-header__background tileBoth header")))
+                    (div (@ (class "page"))
+                         (main (@ (class "page__main"))
+                               (div (@ (class "custom-top"))
+                                    (h1 (@ (class "page-title"))
+                                        "Checking you're not a bot..."))
+                               (div (@ (id "content") #;(class "page-content"))
+                                    (div (@ (id "mw-content-text"))
+                                         (p "To confirm, please click directly in the circle, or hold down the " ,(~a (get-key-solution req)) " key on your keyboard.")
+                                         (noscript (p "JavaScript is required for the captcha. Sorry!"))
+                                         (div (@ (id "captcha-area")))))
+                               ,(application-footer #f)))))))))
+   (when (config-true? 'debug)
+     (xexp->html body))
+   (response/output
+    #:code 200
+    #:headers always-headers
+    (λ (out)
+      (write-html body out)))))
+
+(define (page-captcha-image req)
+  (response-handler
+   (define w (string->number (path/param-path (third (url-path (request-uri req))))))
+   (define h (string->number (path/param-path (fourth (url-path (request-uri req))))))
+   (define-values (at-x at-y) (get-coordinate-solution req w h))
+   (when (config-true? 'captcha::log)
+     (printf "captcha show - size ~a x ~a - solution ~a x ~a [~a]~n" w h at-x at-y (get-ip req)))
+   #;(printf "target: ~a x ~a~ncanvas: ~a x ~a~npict size: ~a-~a ~a-~a~n" at-x at-y x y lt-x rb-x lt-y rb-y)
+   (define dc (make-object bitmap-dc% #f))
+   (send dc set-font font)
+   (define bm (make-object bitmap% w h #f #f))
+   (send dc set-bitmap bm)
+   (draw-pict
+    assembly
+    dc
+    (- at-x inner-x)
+    (- at-y inner-y))
+   (define image (convert bm 'png-bytes))
+   (response/output
+    #:mime-type #"image/png"
+    #:headers (list (header #"Cache-Control" #"no-cache"))
+    (λ (out) (display image out)))))
+
+(define (page-captcha-verify req)
+  (response-handler
+   (match-define (list w h x y) (for/list ([segment (drop (url-path (request-uri req)) 2)])
+                                  (string->number (path/param-path segment))))
+   #;(printf "solution: ~a x ~a~ncoordinate: ~a x ~a~ndist^2: ~a~n" solution-x solution-y x y dist)
+   (define headers
+     (build-headers
+      always-headers
+      (cond
+        [(and (= y 0) (= x (get-key-solution req)))
+         (when (config-true? 'captcha::log)
+           (printf "captcha pass - key ~a [~a]~n" x (get-ip req)))
+
+         (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "key" #:path "/" #:max-age (* 60 60 24 365 10))))]
+        [(= y 0)
+         (when (config-true? 'captcha::log)
+           (printf "captcha fail - key ~a instead of ~a [~a]~n" x (get-key-solution req) (get-ip req)))]
+        [else (void)])
+      (when (> y 0)
+        (let-values ([(solution-x solution-y) (get-coordinate-solution req w h)])
+          (let ([dist (+ (expt (- x solution-x) 2) (expt (- y solution-y) 2))])
+            (cond
+              [(dist . < . (expt (/ diameter 2) 2))
+               (when (config-true? 'captcha::log)
+                 (printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req)))
+               (header #"Set-Cookie" (cookie->set-cookie-header (make-cookie "captcha" "coordinate" #:path "/" #:max-age (* 60 60 24 365 10))))]
+              [else
+               (when (config-true? 'captcha::log)
+                 (printf "captcha pass - coordinate, dist^2 ~a [~a]~n" dist (get-ip req)))]))))))
+   (match (dict-ref (url-query (request-uri req)) 'from #f)
+     [(? string? dest)
+      (response/output
+       #:code 302
+       #:mime-type #"text/plain"
+       #:headers (cons (header #"Location" (string->bytes/utf-8 dest)) headers)
+       (λ (out)
+         (displayln "Checking your answer..." out)))]
+     [#f (next-dispatcher)])))

src/page-category.rkt

@@ -15,11 +15,11 @@
          "application-globals.rkt"
          "config.rkt"
          "data.rkt"
+         "fandom-request.rkt"
          "page-wiki.rkt"
          "../lib/syntax.rkt"
          "../lib/thread-utils.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
@@ -73,30 +73,24 @@
    (define-values (members-data page-data siteinfo)
      (thread-values
       (λ ()
-        (define dest-url
-          (format "~a/api.php?~a"
-                  origin
-                  (params->query `(("action" . "query")
-                                   ("list" . "categorymembers")
-                                   ("cmtitle" . ,prefixed-category)
-                                   ("cmlimit" . "max")
-                                   ("formatversion" . "2")
-                                   ("format" . "json")))))
-        (log-outgoing dest-url)
-        (define dest-res (easy:get dest-url #:timeouts timeouts))
-        (easy:response-json dest-res))
+        (easy:response-json
+         (fandom-get-api
+          wikiname
+          `(("action" . "query")
+            ("list" . "categorymembers")
+            ("cmtitle" . ,prefixed-category)
+            ("cmlimit" . "max")
+            ("formatversion" . "2")
+            ("format" . "json")))))
       (λ ()
-        (define dest-url
-          (format "~a/api.php?~a"
-                  origin
-                  (params->query `(("action" . "parse")
-                                   ("page" . ,prefixed-category)
-                                   ("prop" . "text|headhtml|langlinks")
-                                   ("formatversion" . "2")
-                                   ("format" . "json")))))
-        (log-outgoing dest-url)
-        (define dest-res (easy:get dest-url #:timeouts timeouts))
-        (easy:response-json dest-res))
+        (easy:response-json
+         (fandom-get-api
+          wikiname
+          `(("action" . "parse")
+            ("page" . ,prefixed-category)
+            ("prop" . "text|headhtml|langlinks")
+            ("formatversion" . "2")
+            ("format" . "json")))))
       (λ ()
         (siteinfo-fetch wikiname))))
 

src/page-file.rkt

@@ -15,11 +15,11 @@
          "application-globals.rkt"
          "config.rkt"
          "data.rkt"
+         "fandom-request.rkt"
          "page-wiki.rkt"
          "../lib/syntax.rkt"
          "../lib/thread-utils.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide page-file)
@@ -40,8 +40,7 @@
              (imageDescription . #f))))
 
 (define (url-content-type url)
-  (log-outgoing url)
-  (define dest-res (easy:head url #:timeouts timeouts))
+  (define dest-res (easy:head url))
   (easy:response-headers-ref dest-res 'content-type))
 
 (define (get-media-html url content-type)
@@ -106,20 +105,18 @@
   (response-handler
    (define wikiname (path/param-path (first (url-path (request-uri req)))))
    (define prefixed-title (path/param-path (caddr (url-path (request-uri req)))))
-   (define origin (format "https://~a.fandom.com" wikiname))
-   (define source-url (format "~a/wiki/~a" origin prefixed-title))
+   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title))
 
    (define-values (media-detail siteinfo)
      (thread-values
       (λ ()
-        (define dest-url
-          (format "~a/wikia.php?~a"
-                  origin
-                  (params->query `(("format" . "json") ("controller" . "Lightbox")
-                                                       ("method" . "getMediaDetail")
-                                                       ("fileTitle" . ,prefixed-title)))))
-        (log-outgoing dest-url)
-        (define dest-res (easy:get dest-url #:timeouts timeouts))
+        (define dest-res
+          (fandom-get
+           wikiname
+           (format "/wikia.php?~a"
+                   (params->query `(("format" . "json") ("controller" . "Lightbox")
+                                                        ("method" . "getMediaDetail")
+                                                        ("fileTitle" . ,prefixed-title))))))
         (easy:response-json dest-res))
       (λ ()
         (siteinfo-fetch wikiname))))

src/page-search.rkt

@@ -2,7 +2,6 @@
 (require racket/dict
          racket/list
          racket/string
-         (prefix-in easy: net/http-easy)
          ; html libs
          html-writing
          ; web server libs
@@ -18,15 +17,14 @@
          "../lib/syntax.rkt"
          "../lib/thread-utils.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
  page-search)
 
 (define search-providers
-  (hash "fandom" generate-results-content-fandom
-        "solr" generate-results-content-solr))
+  (hash "fandom" search-fandom
+        "solr" search-solr))
 
 ;; this takes the info we gathered from fandom and makes the big fat x-expression page
 (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f])

src/page-wiki-jsonp.rkt

@@ -0,0 +1,69 @@
+#lang racket/base
+(require racket/list
+         racket/string
+         web-server/http
+         net/url-structs
+         html-writing
+         "application-globals.rkt"
+         "data.rkt"
+         "config.rkt"
+         "../lib/url-utils.rkt"
+         "../lib/xexpr-utils.rkt"
+         "../lib/archive-file-mappings.rkt"
+         "static-data.rkt")
+
+(provide
+ page-wiki-jsonp)
+
+(define (page-wiki-jsonp req)
+  (response-handler
+   (define wikiname (path/param-path (first (url-path (request-uri req)))))
+   (define segments (map path/param-path (cdr (url-path (request-uri req)))))
+   (define path (string-join (cdr segments) "/"))
+   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
+
+   (define wiki-page-script-url
+     (format "https://~a.fandom.com/api.php?~a"
+             wikiname
+             (params->query `(("action" . "parse")
+                              ("page" . ,path)
+                              ("prop" . "text|headhtml|langlinks")
+                              ("formatversion" . "2")
+                              ("format" . "json")
+                              ("callback" . "wikiPageCallback")))))
+   (define siteinfo-script-url
+     (format "https://~a.fandom.com/api.php?~a"
+             wikiname
+             (params->query `(("action" . "query")
+                              ("meta" . "siteinfo")
+                              ("siprop" . "general|rightsinfo")
+                              ("format" . "json")
+                              ("formatversion" . "2")
+                              ("callback" . "siteinfoCallback")))))
+
+   (define body
+     (generate-wiki-page
+      `(div
+        (noscript "You have to enable JavaScript to load wiki pages. Sorry!")
+        (div (@ (id "loading")))
+        (progress (@ (id "progress") (style "margin-bottom: 50vh")))
+        (script ,(format #<<END
+var wikiname = ~v;
+var path = ~v;
+END
+                         wikiname path))
+        (script (@ (src ,(get-static-url "jsonp.js"))))
+        (script (@ (async) (src ,wiki-page-script-url)))
+        (script (@ (async) (src ,siteinfo-script-url))))
+      #:req req
+      #:source-url source-url
+      #:wikiname wikiname
+      #:title (url-segments->guess-title segments)
+      #:siteinfo siteinfo-default))
+   (when (config-true? 'debug)
+     (xexp->html body))
+   (response/output
+    #:code 200
+    #:headers always-headers
+    (λ (out)
+      (write-html body out)))))

src/page-wiki-offline.rkt

@@ -98,16 +98,19 @@
          #:head-data head-data
          #:siteinfo (siteinfo-fetch wikiname)
          ))
-      (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
       (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
+      (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
+      (define redirect-msg-a (if redirect-msg
+                                 ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
+                                 #f))
       (define headers
         (build-headers
          always-headers
          ; redirect-query-parameter: only the string "no" is significant:
          ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
-         (when (and redirect-msg
+         (when (and redirect-msg-a
                     (not (equal? redirect-query-parameter "no")))
-           (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))]
+           (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
                   [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
              (header #"Refresh" value)))))
       (when (config-true? 'debug)
@@ -123,7 +126,8 @@
     ;; page not found on disk, perhaps it's a redirect? redirects are stored in the database
     (var target (query-maybe-value* "select redirect from page where wikiname = ? and basename = ?" wikiname basename))
     [target
-     (generate-redirect (basename->name-for-query target))]
+     ; don't url decode the target, or Category: pages will be interpreted as a protocol
+     (generate-redirect (format "/~a/wiki/~a" wikiname (regexp-replace* #rx"#" target "/")))]
 
     ;; breezewiki doesn't have the page archived, see if we can make a network request for it
     [(not (config-true? 'feature_offline::only))

src/page-wiki.rkt

@@ -6,6 +6,7 @@
          racket/string
          ; libs
          (prefix-in easy: net/http-easy)
+         json
          ; html libs
          "../lib/html-parsing/main.rkt"
          html-writing
@@ -17,17 +18,17 @@
          "application-globals.rkt"
          "config.rkt"
          "data.rkt"
-         "../lib/pure-utils.rkt"
-         "../lib/syntax.rkt"
+         "fandom-request.rkt"
+         "../lib/archive-file-mappings.rkt"
          "../lib/thread-utils.rkt"
          "../lib/tree-updater.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
  ; used by the web server
  page-wiki
+ page-wiki-with-data
  ; used by page-category, and similar pages that are partially wiki pages
  update-tree-wiki
  preprocess-html-wiki)
@@ -37,67 +38,114 @@
 
 (define (page-wiki req)
   (define wikiname (path/param-path (first (url-path (request-uri req)))))
+  (define segments (map path/param-path (cdr (url-path (request-uri req)))))
   (define user-cookies (user-cookies-getter req))
-  (define origin (format "https://~a.fandom.com" wikiname))
-  (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
+  (define path (string-join (cdr segments) "/"))
   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
 
   (define-values (dest-res siteinfo)
     (thread-values
      (λ ()
-       (define dest-url
-         (format "~a/api.php?~a"
-                 origin
-                 (params->query `(("action" . "parse")
-                                  ("page" . ,path)
-                                  ("prop" . "text|headhtml|langlinks")
-                                  ("formatversion" . "2")
-                                  ("format" . "json")))))
-       (log-outgoing dest-url)
-       (easy:get dest-url
-                 #:timeouts timeouts
-                 #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
+       (fandom-get-api
+        wikiname
+        `(("action" . "parse")
+          ("page" . ,path)
+          ("prop" . "text|headhtml|langlinks")
+          ("formatversion" . "2")
+          ("format" . "json"))
+        #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
      (λ ()
        (siteinfo-fetch wikiname))))
 
   (cond
     [(eq? 200 (easy:response-status-code dest-res))
-     (let* ([data (easy:response-json dest-res)]
-            [title (jp "/parse/title" data "")]
-            [page-html (jp "/parse/text" data "")]
-            [page-html (preprocess-html-wiki page-html)]
-            [page (html->xexp page-html)]
-            [head-data ((head-data-getter wikiname) data)])
+     (let ([data (easy:response-json dest-res)])
        (if (equal? "missingtitle" (jp "/error/code" data #f))
            (next-dispatcher)
-           (response-handler
-            (define body
-              (generate-wiki-page
-               (update-tree-wiki page wikiname)
-               #:req req
-               #:source-url source-url
-               #:wikiname wikiname
-               #:title title
-               #:head-data head-data
-               #:siteinfo siteinfo))
-            (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
-            (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
-            (define headers
-              (build-headers
-               always-headers
-               ; redirect-query-parameter: only the string "no" is significant:
-               ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
-               (when (and redirect-msg
-                          (not (equal? redirect-query-parameter "no")))
-                 (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))]
-                        [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
-                   (header #"Refresh" value)))))
-            (when (config-true? 'debug)
-              ; used for its side effects
-              ; convert to string with error checking, error will be raised if xexp is invalid
-              (xexp->html body))
-            (response/output
-             #:code 200
-             #:headers headers
-             (λ (out)
-               (write-html body out))))))]))
+           (take-json-rewrite-and-return-page data)))]
+    [(eq? 404 (easy:response-status-code dest-res))
+     (next-dispatcher)]
+    [(memq (easy:response-status-code dest-res) '(403 406))
+     (response-handler
+      (define body
+        (generate-wiki-page
+         `(div
+           (p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.")
+           (p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.")
+           (p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom."))))
+         #:req req
+         #:source-url source-url
+         #:wikiname wikiname
+         #:title (url-segments->guess-title segments)
+         #:siteinfo siteinfo))
+      (response/output
+       #:code 503
+       #:headers (build-headers
+                  always-headers
+                  (header #"Retry-After" #"30")
+                  (header #"Cache-Control" #"max-age=30, public")
+                  (header #"Refresh" #"35"))
+       (λ (out)
+         (write-html body out))))]
+    [else
+     (response-handler
+      (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
+             wikiname
+             path
+             (easy:response-status-code dest-res)
+             (easy:response-body dest-res)))]))
+
+(define (page-wiki-with-data req)
+  (response-handler
+   (define post-data/bytes (request-post-data/raw req))
+   (when (not post-data/bytes)
+     (raise-user-error 'page-wiki-with-data "POST requests only, please."))
+   (define post-data/string (bytes->string/utf-8 post-data/bytes))
+   (define post-data (string->jsexpr post-data/string))
+   (define wikiname (jp "/wikiname" post-data))
+   (define path (jp "/path" post-data))
+   (take-json-rewrite-and-return-page
+    #:req req
+    #:wikiname wikiname
+    #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
+    #:data (jp "/data" post-data)
+    #:siteinfo (data->siteinfo (jp "/siteinfo" post-data)))))
+
+(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo)
+  (define title (jp "/parse/title" data ""))
+  (define page-html (preprocess-html-wiki (jp "/parse/text" data "")))
+  (define page (html->xexp page-html))
+  (define head-data ((head-data-getter wikiname) data))
+  (response-handler
+   (define body
+     (generate-wiki-page
+      (update-tree-wiki page wikiname)
+      #:req req
+      #:source-url source-url
+      #:wikiname wikiname
+      #:title title
+      #:head-data head-data
+      #:siteinfo siteinfo))
+   (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
+   (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
+   (define redirect-msg-a (if redirect-msg
+                              ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
+                              #f))
+   (define html (xexp->html-bytes body))
+   (define headers
+     (build-headers
+      always-headers
+      ; redirect-query-parameter: only the string "no" is significant:
+      ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
+      (when (and redirect-msg-a
+                 (not (equal? redirect-query-parameter "no")))
+        (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
+               [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
+          (header #"Refresh" value)))))
+   (response/full
+    200
+    #"OK"
+    (current-seconds)
+    #"text/html; charset=utf-8"
+    headers
+    (list html))))

src/search-provider-fandom.rkt

@@ -3,38 +3,34 @@
          (prefix-in easy: net/http-easy)
          "application-globals.rkt"
          "config.rkt"
+         "fandom-request.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
- generate-results-content-fandom)
+ search-fandom)
 
 (module+ test
   (require rackunit
            "test-utils.rkt")
-  (define search-json-data
-    '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))))))
+  (define search-results-data
+    '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
 
-(define (generate-results-content-fandom wikiname query params)
-  ;; constructing the URL where I want to get fandom data from...
-  (define origin (format "https://~a.fandom.com" wikiname))
-  ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json
-  (define dest-url
-    (format "~a/api.php?~a"
-            origin
-            (params->query `(("action" . "query")
-                             ("list" . "search")
-                             ("srsearch" . ,query)
-                             ("formatversion" . "2")
-                             ("format" . "json")))))
-  ;; HTTP request to dest-url for search results
-  (log-outgoing dest-url)
-  (define res (easy:get dest-url #:timeouts timeouts))
+(define (search-fandom wikiname query params)
+  (define res
+    (fandom-get-api
+     wikiname
+     `(("action" . "query")
+       ("list" . "search")
+       ("srsearch" . ,query)
+       ("formatversion" . "2")
+       ("format" . "json"))))
   (define json (easy:response-json res))
   (define search-results (jp "/query/search" json))
+  (generate-results-content-fandom wikiname query search-results))
 
-  ;; generate content for display in the wiki page layout
+;;; generate content for display in the wiki page layout
+(define (generate-results-content-fandom wikiname query search-results)
   `(div (@ (class "mw-parser-output"))
         ;; header before the search results showing how many we found
         (p ,(format "~a results found for " (length search-results))
@@ -60,4 +56,4 @@
 (module+ test
   (parameterize ([(config-parameter 'feature_offline::only) "false"])
     (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")
-                                      (generate-results-content-fandom test-req "" "test" "Gacha" search-json-data))))))
+                                      (generate-results-content-fandom "test" "Gacha" search-results-data))))))

src/search-provider-solr.rkt

@@ -5,15 +5,14 @@
          "application-globals.rkt"
          "../lib/html-parsing/main.rkt"
          "../lib/url-utils.rkt"
-         "whole-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
- generate-results-content-solr)
+ search-solr)
 
 (struct result^ (hl-title hl-body kb words page-path) #:transparent)
 
-(define (generate-results-content-solr wikiname query params)
+(define (search-solr wikiname query params)
   ;; grab things from params that would modify the search
   (define op (if (equal? (dict-ref params 'op #f) "or") '("or" . "OR") '("and" . "AND")))
   (define sort (if (equal? (dict-ref params 'sort #f) "len") '("len" . "len desc") '("relevance" . "score desc")))
@@ -36,8 +35,7 @@
                              ("hl.tag.post" . "</mark>")
                              ("sort" . ,(cdr sort))))))
   ;; HTTP request to dest-url for search results
-  (log-outgoing dest-url)
-  (define res (easy:get dest-url #:timeouts timeouts))
+  (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5)))
   (define json (easy:response-json res))
 
   ;; build result objects

src/whole-utils.rkt

@@ -1,11 +0,0 @@
-#lang typed/racket/base
-(require "config.rkt")
-
-(provide
- ; prints "out: <url>"
- log-outgoing)
-
-(: log-outgoing (String -> Void))
-(define (log-outgoing url-string)
-  (when (config-true? 'log_outgoing)
-    (printf "out: ~a~n" url-string)))

static/captcha.js

@@ -0,0 +1,24 @@
+const u = new URL(location)
+const from = u.searchParams.get("from") || location.href
+let answered = false
+
+const area = document.getElementById("captcha-area")
+const areaBox = area.getBoundingClientRect()
+const width = Math.floor(areaBox.width)
+const height = Math.floor(window.innerHeight - areaBox.bottom - areaBox.left)
+const img = document.createElement("img")
+img.src = `/captcha/img/${width}/${height}`
+img.addEventListener("click", event => {
+	if (answered) return
+	answered = true
+	location = `/captcha/verify/${width}/${height}/${event.offsetX}/${event.offsetY}?` + new URLSearchParams({from})
+})
+area.appendChild(img)
+
+document.addEventListener("keydown", event => {
+	if (event.repeat) {
+		if (answered) return
+		answered = true
+		location = `/captcha/verify/0/0/${event.key}/0?` + new URLSearchParams({from})
+	}
+})

static/jsonp.js

@@ -0,0 +1,59 @@
+const loading = document.getElementById("loading")
+loading.textContent = "Loading, please wait..."
+const progress = document.getElementById("progress")
+
+let wikiPage = null
+function wikiPageCallback(data) {
+	wikiPage = data
+	cont()
+}
+
+let siteinfo = null
+function siteinfoCallback(data) {
+	siteinfo = data
+	cont()
+}
+
+async function cont() {
+	if (!(wikiPage && siteinfo)) return
+
+	const xhr = new XMLHttpRequest();
+
+	const uploadFraction = 0.7
+
+	// Upload progress
+	xhr.upload.addEventListener("progress", event => {
+		if (event.lengthComputable) {
+			progress.value = (event.loaded / event.total) * uploadFraction
+		}
+	})
+
+	// Download progress
+	xhr.addEventListener("progress", event => {
+		if (event.lengthComputable) {
+			progress.value = (event.loaded / event.total) * (1 - uploadFraction) + uploadFraction
+		}
+	})
+
+	xhr.addEventListener("load", () => {
+		console.log(xhr)
+		const imported = document.importNode(xhr.responseXML.getElementById("content"), true)
+		document.getElementById("content").replaceWith(imported)
+		document.title = xhr.responseXML.title
+		for (const e of xhr.responseXML.head.children) {
+			if (["LINK"].includes(e.tagName)) {
+				const imported = document.importNode(e, true)
+				document.head.appendChild(imported)
+			}
+		}
+	})
+
+	xhr.open("POST", "/api/render/wiki")
+	xhr.responseType = "document"
+	xhr.send(JSON.stringify({
+		data: wikiPage,
+		siteinfo,
+		wikiname,
+		path
+	}));
+}

static/main.css

@@ -202,11 +202,11 @@ figcaption, .lightbox-caption, .thumbcaption {
     padding: 0;
 }
 
-/* show tabs always */
-.wds-tabs__wrapper {
+/* show tabs if tabs.js isn't loaded */
+.bw-tabs-nojs .wds-tabs__wrapper {
     display: none;
 }
-.wds-tab__content {
+.bw-tabs-nojs .wds-tab__content {
     display: block;
 }
 
@@ -431,6 +431,25 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */
     font-size: 14px;
     text-align: right;
 }
+/* more compact notice after it's been seen the first time */
+.niwa--seen {
+    padding: 1.5vw 2vw 2vw;
+    overflow-y: auto;
+    max-height: min(280px, 33vh);
+    font-size: 17px;
+    margin-top: -2vw;
+    margin-bottom: 12px;
+}
+.niwa--seen .niwa__header {
+    font-size: 26px;
+}
+.niwa--seen .niwa__go {
+    padding: 10px 18px;
+    font-size: 20px;
+}
+.niwa--seen .niwa__got-it {
+    display: none;
+}
 
 /* media queries */
 

static/robots.txt

@@ -2,3 +2,4 @@ User-Agent: *
 Disallow: /*/wiki/*
 Disallow: /proxy
 Disallow: /set-user-settings
+Disallow: /captcha

static/tabs.js

@@ -0,0 +1,40 @@
+"use strict";
+
+const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null
+
+for (const tabber of document.body.querySelectorAll(".wds-tabber")) {
+	for (const [tab, content] of getTabberTabs(tabber)) {
+		// set up click listener on every tab
+		tab.addEventListener("click", e => {
+			setCurrentTab(tabber, tab, content)
+			e.preventDefault()
+		})
+
+		// re-open a specific tab on page load based on the URL hash
+		if (tab.dataset.hash === tabFromHash) {
+			setCurrentTab(tabber, tab, content)
+			tab.scrollIntoView()
+		}
+	}
+}
+
+function getTabberTabs(tabber) {
+	// need to scope the selector to handle nested tabs. see /unturned/wiki/Crate for an example
+	const tabs = [...tabber.querySelectorAll(":scope > .wds-tabs__wrapper .wds-tabs__tab")]
+	const contents = [...tabber.querySelectorAll(":scope > .wds-tab__content")]
+	return tabs.map((_, index) => [tabs[index], contents[index]]) // transpose arrays into [[tab, content], ...]
+}
+
+function setCurrentTab(tabber, tab, content) {
+	// clear currently selected tab
+	getTabberTabs(tabber).flat().forEach(e => e.classList.remove("wds-is-current"))
+
+	// select new tab
+	tab.classList.add("wds-is-current")
+	content.classList.add("wds-is-current")
+	if (tab.dataset.hash) {
+		history.replaceState(null, "", `#${tab.dataset.hash}`)
+	}
+}
+
+document.body.classList.remove("bw-tabs-nojs")