From 9773e62c46b3faebb8e621718f3e9870fbd24d10 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 29 May 2023 23:02:28 +0700 Subject: [PATCH 01/14] Add better support for tabs Some pages break without actual tab support, such as https://breezewiki.com/ben10/wiki/Ultimatrix_(Original)#Modes This change aims to work with old browsers (such as Firefox for Android 68) and browsers with Javascript disabled (by showing all tab contents and hiding the tab bar, i.e. how tabs work before this change). --- src/application-globals.rkt | 3 +- static/main.css | 6 ++-- static/tabs.js | 63 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 static/tabs.js diff --git a/src/application-globals.rkt b/src/application-globals.rkt index de60820..a413785 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -203,10 +203,11 @@ `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js")))) "") (script (@ (type "module") (src ,(get-static-url "countdown.js")))) + (script (@ (defer) (src ,(get-static-url "tabs.js")))) (link (@ (rel "icon") (href ,(u (λ (v) (config-true? 'strict_proxy)) (λ (v) (u-proxy-url v)) (head-data^-icon-url head-data)))))) - (body (@ (class ,(head-data^-body-class head-data))) + (body (@ (class ,(head-data^-body-class head-data) " bw-tabs-nojs")) ,(let ([extension-eligible? (cond/var [(not req) #f] diff --git a/static/main.css b/static/main.css index cb32df5..6b07d2b 100644 --- a/static/main.css +++ b/static/main.css @@ -202,11 +202,11 @@ figcaption, .lightbox-caption, .thumbcaption { padding: 0; } -/* show tabs always */ -.wds-tabs__wrapper { +/* show tabs if tabs.js isn't loaded */ +.bw-tabs-nojs .wds-tabs__wrapper { display: none; } -.wds-tab__content { +.bw-tabs-nojs .wds-tab__content { display: block; } diff --git a/static/tabs.js b/static/tabs.js new file mode 100644 index 0000000..589261d --- /dev/null +++ b/static/tabs.js @@ -0,0 +1,63 @@ +"use strict"; + +function handleTabber(tabber) { + let [tabs, contents] = getTabs(tabber); + + for (let i in tabs) { + let tab = tabs[i]; + let content = contents[i]; + tab.addEventListener("click", function(e) { + let [currentTab, currentContent] = getCurrentTab(tabber); + if (currentTab) { + currentTab.classList.remove("wds-is-current"); + } + if (currentContent) { + currentContent.classList.remove("wds-is-current"); + } + + tab.classList.add("wds-is-current"); + content.classList.add("wds-is-current"); + e.preventDefault(); + }); + } +} + +for (let tabber of document.body.querySelectorAll(".wds-tabber")) { + handleTabber(tabber); +} +document.body.classList.remove("bw-tabs-nojs"); + + + +function getTabs(tabber) { + let tabs = []; + let contents = []; + + for (let i of tabber.querySelector(".wds-tabs__wrapper").querySelectorAll(".wds-tabs__tab")) { + tabs.push(i); + } + for (let i of tabber.children) { + if (!i.matches(".wds-tab__content")) { + continue; + } + contents.push(i); + } + + return [tabs, contents]; +} + +function getCurrentTab(tabber) { + let tab = null; + let content = null; + + tab = tabber.querySelector(".wds-tabs__wrapper").querySelector(".wds-tabs__tab.wds-is-current"); + for (let i of tabber.children) { + if (!i.matches(".wds-tab__content.wds-is-current")) { + continue; + } + content = i; + break; + } + + return [tab, content]; +} From ead6896818be96048138d0d3ecd12438f8d475b8 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 30 May 2023 13:41:49 +0700 Subject: [PATCH 02/14] Add the ability to specify/open the last open tab in the URL --- static/tabs.js | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/static/tabs.js b/static/tabs.js index 589261d..916cb4d 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -1,30 +1,23 @@ "use strict"; -function handleTabber(tabber) { +let tabToFind = location.hash.length > 1 ? location.hash.substring(1) : null; +for (let tabber of document.body.querySelectorAll(".wds-tabber")) { let [tabs, contents] = getTabs(tabber); for (let i in tabs) { let tab = tabs[i]; let content = contents[i]; - tab.addEventListener("click", function(e) { - let [currentTab, currentContent] = getCurrentTab(tabber); - if (currentTab) { - currentTab.classList.remove("wds-is-current"); - } - if (currentContent) { - currentContent.classList.remove("wds-is-current"); - } - tab.classList.add("wds-is-current"); - content.classList.add("wds-is-current"); + tab.addEventListener("click", function(e) { + setCurrentTab(tabber, tab, content); e.preventDefault(); }); + if (tab.dataset.hash === tabToFind) { + setCurrentTab(tabber, tab, content); + tabToFind = null; + } } } - -for (let tabber of document.body.querySelectorAll(".wds-tabber")) { - handleTabber(tabber); -} document.body.classList.remove("bw-tabs-nojs"); @@ -61,3 +54,18 @@ function getCurrentTab(tabber) { return [tab, content]; } + +function setCurrentTab(tabber, tab, content) { + let [currentTab, currentContent] = getCurrentTab(tabber); + if (currentTab) { + currentTab.classList.remove("wds-is-current"); + } + if (currentContent) { + currentContent.classList.remove("wds-is-current"); + } + + tab.classList.add("wds-is-current"); + content.classList.add("wds-is-current"); + location.hash = "#" + tab.dataset.hash; + history.pushState(null, "", "#" + tab.dataset.hash); +} From f5399524b135eedf346ad758a88f37cba57be700 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 30 May 2023 14:01:14 +0700 Subject: [PATCH 03/14] Prevent linking to tabs with no IDs --- static/tabs.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/static/tabs.js b/static/tabs.js index 916cb4d..6f582e8 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -14,7 +14,6 @@ for (let tabber of document.body.querySelectorAll(".wds-tabber")) { }); if (tab.dataset.hash === tabToFind) { setCurrentTab(tabber, tab, content); - tabToFind = null; } } } @@ -66,6 +65,8 @@ function setCurrentTab(tabber, tab, content) { tab.classList.add("wds-is-current"); content.classList.add("wds-is-current"); - location.hash = "#" + tab.dataset.hash; - history.pushState(null, "", "#" + tab.dataset.hash); + if (tab.dataset.hash) { + location.hash = "#" + tab.dataset.hash; + history.pushState(null, "", "#" + tab.dataset.hash); + } } From dcb8a8a590df66023cc355266cc1fc0c4f278438 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 6 Nov 2023 20:31:20 +1100 Subject: [PATCH 04/14] Prevent making duplicate history entries --- static/tabs.js | 1 - 1 file changed, 1 deletion(-) diff --git a/static/tabs.js b/static/tabs.js index 6f582e8..4c4cdda 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -67,6 +67,5 @@ function setCurrentTab(tabber, tab, content) { content.classList.add("wds-is-current"); if (tab.dataset.hash) { location.hash = "#" + tab.dataset.hash; - history.pushState(null, "", "#" + tab.dataset.hash); } } From 2b3a8fe1084abab995f6d0937f673dcb4eac78b6 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 13 Nov 2023 14:35:07 +1100 Subject: [PATCH 05/14] Fix scrolling to sections if a tab's hash coincides with one ben10/wiki/Alien_X_(Classic)#Appearances --- static/tabs.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/tabs.js b/static/tabs.js index 4c4cdda..a077efe 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -66,6 +66,9 @@ function setCurrentTab(tabber, tab, content) { tab.classList.add("wds-is-current"); content.classList.add("wds-is-current"); if (tab.dataset.hash) { - location.hash = "#" + tab.dataset.hash; + let fragment = "#" + tab.dataset.hash; + if (location.hash !== fragment) { + history.pushState(null, "", fragment); + } } } From b02e2a405329ec56c5ce8fd357fd60ff11332033 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 11:21:42 +1300 Subject: [PATCH 06/14] Fix failing test after search was refactored --- src/page-search.rkt | 4 ++-- src/search-provider-fandom.rkt | 14 ++++++++------ src/search-provider-solr.rkt | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/page-search.rkt b/src/page-search.rkt index 193dbd6..019ebfe 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -25,8 +25,8 @@ page-search) (define search-providers - (hash "fandom" generate-results-content-fandom - "solr" generate-results-content-solr)) + (hash "fandom" search-fandom + "solr" search-solr)) ;; this takes the info we gathered from fandom and makes the big fat x-expression page (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f]) diff --git a/src/search-provider-fandom.rkt b/src/search-provider-fandom.rkt index 945b111..2338c13 100644 --- a/src/search-provider-fandom.rkt +++ b/src/search-provider-fandom.rkt @@ -8,15 +8,15 @@ "../lib/xexpr-utils.rkt") (provide - generate-results-content-fandom) + search-fandom) (module+ test (require rackunit "test-utils.rkt") - (define search-json-data - '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) + (define search-results-data + '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) -(define (generate-results-content-fandom wikiname query params) +(define (search-fandom wikiname query params) ;; constructing the URL where I want to get fandom data from... (define origin (format "https://~a.fandom.com" wikiname)) ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json @@ -33,8 +33,10 @@ (define res (easy:get dest-url #:timeouts timeouts)) (define json (easy:response-json res)) (define search-results (jp "/query/search" json)) + (generate-results-content-fandom wikiname query search-results)) - ;; generate content for display in the wiki page layout +;;; generate content for display in the wiki page layout +(define (generate-results-content-fandom wikiname query search-results) `(div (@ (class "mw-parser-output")) ;; header before the search results showing how many we found (p ,(format "~a results found for " (length search-results)) @@ -60,4 +62,4 @@ (module+ test (parameterize ([(config-parameter 'feature_offline::only) "false"]) (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") - (generate-results-content-fandom test-req "" "test" "Gacha" search-json-data)))))) + (generate-results-content-fandom "test" "Gacha" search-results-data)))))) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index ed9fb7b..1ec48e2 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -9,11 +9,11 @@ "../lib/xexpr-utils.rkt") (provide - generate-results-content-solr) + search-solr) (struct result^ (hl-title hl-body kb words page-path) #:transparent) -(define (generate-results-content-solr wikiname query params) +(define (search-solr wikiname query params) ;; grab things from params that would modify the search (define op (if (equal? (dict-ref params 'op #f) "or") '("or" . "OR") '("and" . "AND"))) (define sort (if (equal? (dict-ref params 'sort #f) "len") '("len" . "len desc") '("relevance" . "score desc"))) From a52d131b936037fdd7847bf8d58c1662227101ad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 11 Jan 2024 22:33:59 +1300 Subject: [PATCH 07/14] Split massive uploads in Solr indexer --- archiver/fts.rkt | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/archiver/fts.rkt b/archiver/fts.rkt index c2f597b..6a00041 100644 --- a/archiver/fts.rkt +++ b/archiver/fts.rkt @@ -8,6 +8,7 @@ racket/promise racket/port racket/runtime-path + racket/sequence racket/string file/gunzip db @@ -156,8 +157,16 @@ (define data (cond [(and (read-from-cache?) (file-exists? "cache.rkt")) - (displayln "Reading in...") - (with-input-from-file "cache.rkt" (λ () (read)))] + (define size (file-size "cache.rkt")) + (call-with-input-file "cache.rkt" + (λ (in) + (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position in) 64 1024)) + (ceiling (/ size 64 1024)) + "Reading in...")) + 2)) + (begin0 + (read in) + (quit))))] [else (define x (box (progress^ 0 1 "..."))) (define quit (make-progress (λ () (unbox x)))) @@ -183,18 +192,22 @@ (display "Converting... ") (flush-output) - (define ser (jsexpr->bytes data)) - (define ser-port (open-input-bytes ser)) - (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) - (ceiling (/ (bytes-length ser) 64 1024)) - "Posting...")) - 2)) - (define res - (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) - #:data ser-port - #:headers '#hasheq((Content-Type . "application/json")) - #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) - (quit) - (displayln (response-status-line res))) + (define slice-size 30000) + (define slices (ceiling (/ (length data) slice-size))) + (for ([slice (in-slice slice-size data)] + [i (in-naturals 1)]) + (define ser (jsexpr->bytes slice)) + (define ser-port (open-input-bytes ser)) + (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) + (ceiling (/ (bytes-length ser) 64 1024)) + (format "Posting... (~a/~a)" i slices))) + 2)) + (define res + (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) + #:data ser-port + #:headers '#hasheq((Content-Type . "application/json")) + #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) + (quit) + (displayln (response-status-line res)))) (run start) From 6260ba809bdd2321fa59c6133f749f2db0d23e17 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 1 May 2024 00:53:09 +1200 Subject: [PATCH 08/14] Fix running out of file descriptors --- lib/thread-utils.rkt | 3 +-- src/application-globals.rkt | 3 --- src/data.rkt | 20 +++++++------- src/dispatcher-tree.rkt | 21 +++++++++++++-- src/fandom-request.rkt | 48 ++++++++++++++++++++++++++++++++++ src/page-category.rkt | 42 +++++++++++++---------------- src/page-file.rkt | 23 +++++++--------- src/page-search.rkt | 2 -- src/page-wiki.rkt | 34 +++++++++++++----------- src/search-provider-fandom.rkt | 24 +++++++---------- src/search-provider-solr.rkt | 3 +-- src/whole-utils.rkt | 11 -------- 12 files changed, 134 insertions(+), 100 deletions(-) create mode 100644 src/fandom-request.rkt delete mode 100644 src/whole-utils.rkt diff --git a/lib/thread-utils.rkt b/lib/thread-utils.rkt index f907dac..66e2b4c 100644 --- a/lib/thread-utils.rkt +++ b/lib/thread-utils.rkt @@ -1,6 +1,5 @@ #lang racket/base -(require (prefix-in easy: net/http-easy) - "../src/data.rkt" +(require "../src/data.rkt" "xexpr-utils.rkt") (provide diff --git a/src/application-globals.rkt b/src/application-globals.rkt index de60820..fb8f118 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -22,8 +22,6 @@ (provide ; headers to always send on all http responses always-headers - ; timeout durations for http-easy requests - timeouts ; generates a consistent footer application-footer ; generates a consistent template for wiki page content to sit in @@ -39,7 +37,6 @@ (define always-headers (list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom (header #"Link" (string->bytes/latin-1 link-header)))) -(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define-runtime-path path-static "../static") (define theme-icons diff --git a/src/data.rkt b/src/data.rkt index b22e8a0..6975b37 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -7,8 +7,8 @@ (prefix-in easy: net/http-easy) db memo + "fandom-request.rkt" "static-data.rkt" - "whole-utils.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt" "../archiver/archiver-database.rkt" @@ -54,16 +54,14 @@ (vector-ref row 3))) siteinfo-default)] [else - (define dest-url - (format "https://~a.fandom.com/api.php?~a" - wikiname - (params->query '(("action" . "query") - ("meta" . "siteinfo") - ("siprop" . "general|rightsinfo") - ("format" . "json") - ("formatversion" . "2"))))) - (log-outgoing dest-url) - (define res (easy:get dest-url)) + (define res + (fandom-get-api + wikiname + (params->query '(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2"))))) (define data (easy:response-json res)) (siteinfo^ (jp "/query/general/sitename" data) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 48e8ebb..0212242 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -33,12 +33,29 @@ ; don't forget that I'm returning *code* - return a call to the function (datum->syntax stx `(make-dispatcher-tree ,ds))) +; guard that the page returned a response, otherwise print more detailed debugging information +(define-syntax-rule (page ds name) + (λ (req) + (define dispatcher (hash-ref ds (quote name))) + (define page-response (dispatcher req)) + (if (response? page-response) + page-response + (response/output + #:code 500 + #:mime-type #"text/plain" + (λ (out) + (for ([port (list (current-error-port) out)]) + (parameterize ([current-output-port port]) + (printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n" + (quote name) + page-response)))))))) + (define (make-dispatcher-tree ds) (define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher)) (define tree (sequencer:make subdomain-dispatcher - (pathprocedure:make "/" (hash-ref ds 'page-home)) + (pathprocedure:make "/" (page ds page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) @@ -48,7 +65,7 @@ (if (config-true? 'feature_offline::enabled) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) - (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (if (config-true? 'feature_offline::enabled) diff --git a/src/fandom-request.rkt b/src/fandom-request.rkt new file mode 100644 index 0000000..966eeee --- /dev/null +++ b/src/fandom-request.rkt @@ -0,0 +1,48 @@ +#lang typed/racket/base +(require "config.rkt" + "../lib/url-utils.rkt") +(define-type Headers (HashTable Symbol (U Bytes String))) +(require/typed net/http-easy + [#:opaque Timeout-Config timeout-config?] + [#:opaque Response response?] + [#:opaque Session session?] + [current-session (Parameter Session)] + [make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)] + [get ((U Bytes String) + [#:close? Boolean] + [#:headers Headers] + [#:timeouts Timeout-Config] + [#:max-attempts Exact-Positive-Integer] + [#:max-redirects Exact-Nonnegative-Integer] + [#:user-agent (U Bytes String)] + -> Response)]) + +(provide + fandom-get + fandom-get-api + timeouts) + +(define timeouts (make-timeout-config #:lease 5 #:connect 5)) + +(: no-headers Headers) +(define no-headers '#hasheq()) + +(: fandom-get (String String [#:headers (Option Headers)] -> Response)) +(define (fandom-get wikiname path #:headers [headers #f]) + (define dest-url (string-append "https://www.fandom.com" path)) + (define host (string-append wikiname ".fandom.com")) + (log-outgoing wikiname path) + (get dest-url + #:timeouts timeouts + #:headers (hash-set (or headers no-headers) 'Host host))) + +(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response)) +(define (fandom-get-api wikiname params #:headers [headers #f]) + (fandom-get wikiname + (string-append "/api.php?" (params->query params)) + #:headers headers)) + +(: log-outgoing (String String -> Void)) +(define (log-outgoing wikiname path) + (when (config-true? 'log_outgoing) + (printf "out: ~a ~a~n" wikiname path))) diff --git a/src/page-category.rkt b/src/page-category.rkt index 213d423..e1fe659 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -15,11 +15,11 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "page-wiki.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -73,30 +73,24 @@ (define-values (members-data page-data siteinfo) (thread-values (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) + (easy:response-json + (fandom-get-api + wikiname + `(("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))))) (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,prefixed-category) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) + (easy:response-json + (fandom-get-api + wikiname + `(("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))))) (λ () (siteinfo-fetch wikiname)))) diff --git a/src/page-file.rkt b/src/page-file.rkt index 2a7332c..5151f1d 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -15,11 +15,11 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "page-wiki.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide page-file) @@ -40,8 +40,7 @@ (imageDescription . #f)))) (define (url-content-type url) - (log-outgoing url) - (define dest-res (easy:head url #:timeouts timeouts)) + (define dest-res (easy:head url)) (easy:response-headers-ref dest-res 'content-type)) (define (get-media-html url content-type) @@ -106,20 +105,18 @@ (response-handler (define wikiname (path/param-path (first (url-path (request-uri req))))) (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-title)) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title)) (define-values (media-detail siteinfo) (thread-values (λ () - (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . ,prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) + (define dest-res + (fandom-get + wikiname + (format "/wikia.php?~a" + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title)))))) (easy:response-json dest-res)) (λ () (siteinfo-fetch wikiname)))) diff --git a/src/page-search.rkt b/src/page-search.rkt index 019ebfe..39f361a 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -2,7 +2,6 @@ (require racket/dict racket/list racket/string - (prefix-in easy: net/http-easy) ; html libs html-writing ; web server libs @@ -18,7 +17,6 @@ "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index ae060d4..f16792c 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -17,12 +17,12 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "../lib/pure-utils.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/tree-updater.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -38,25 +38,20 @@ (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) (define user-cookies (user-cookies-getter req)) - (define origin (format "https://~a.fandom.com" wikiname)) (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define-values (dest-res siteinfo) (thread-values (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,path) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (easy:get dest-url - #:timeouts timeouts - #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) + (fandom-get-api + wikiname + `(("action" . "parse") + ("page" . ,path) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json")) + #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) (λ () (siteinfo-fetch wikiname)))) @@ -103,4 +98,13 @@ #:code 200 #:headers headers (λ (out) - (write-html body out))))))])) + (write-html body out))))))] + [(eq? 404 (easy:response-status-code dest-res)) + (next-dispatcher)] + [else + (response-handler + (error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" + wikiname + path + (easy:response-status-code dest-res) + (easy:response-body dest-res)))])) diff --git a/src/search-provider-fandom.rkt b/src/search-provider-fandom.rkt index 2338c13..b8dd48f 100644 --- a/src/search-provider-fandom.rkt +++ b/src/search-provider-fandom.rkt @@ -3,8 +3,8 @@ (prefix-in easy: net/http-easy) "application-globals.rkt" "config.rkt" + "fandom-request.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -17,20 +17,14 @@ '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) (define (search-fandom wikiname query params) - ;; constructing the URL where I want to get fandom data from... - (define origin (format "https://~a.fandom.com" wikiname)) - ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "search") - ("srsearch" . ,query) - ("formatversion" . "2") - ("format" . "json"))))) - ;; HTTP request to dest-url for search results - (log-outgoing dest-url) - (define res (easy:get dest-url #:timeouts timeouts)) + (define res + (fandom-get-api + wikiname + `(("action" . "query") + ("list" . "search") + ("srsearch" . ,query) + ("formatversion" . "2") + ("format" . "json")))) (define json (easy:response-json res)) (define search-results (jp "/query/search" json)) (generate-results-content-fandom wikiname query search-results)) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index 1ec48e2..31813da 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -5,7 +5,6 @@ "application-globals.rkt" "../lib/html-parsing/main.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -37,7 +36,7 @@ ("sort" . ,(cdr sort)))))) ;; HTTP request to dest-url for search results (log-outgoing dest-url) - (define res (easy:get dest-url #:timeouts timeouts)) + (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5))) (define json (easy:response-json res)) ;; build result objects diff --git a/src/whole-utils.rkt b/src/whole-utils.rkt deleted file mode 100644 index 7118866..0000000 --- a/src/whole-utils.rkt +++ /dev/null @@ -1,11 +0,0 @@ -#lang typed/racket/base -(require "config.rkt") - -(provide - ; prints "out: " - log-outgoing) - -(: log-outgoing (String -> Void)) -(define (log-outgoing url-string) - (when (config-true? 'log_outgoing) - (printf "out: ~a~n" url-string))) From 7dff049ece876d1d31d4537c2176629cabdd7af0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 1 May 2024 00:57:13 +1200 Subject: [PATCH 09/14] Wrap all pages in response safety checker --- src/dispatcher-tree.rkt | 20 ++++++++++---------- src/search-provider-solr.rkt | 1 - 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 0212242..a967095 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -56,20 +56,20 @@ (sequencer:make subdomain-dispatcher (pathprocedure:make "/" (page ds page-home)) - (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) - (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) - (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works)) - (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) - (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) + (pathprocedure:make "/proxy" (page ds page-proxy)) + (pathprocedure:make "/search" (page ds page-global-search)) + (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings)) + (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works)) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category))) + (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file))) (if (config-true? 'feature_offline::enabled) - (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) - (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) - (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home))) (if (config-true? 'feature_offline::enabled) - (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive))) + (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive))) (λ (_conn _req) (next-dispatcher))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found)))) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index 31813da..c15e31f 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -35,7 +35,6 @@ ("hl.tag.post" . "") ("sort" . ,(cdr sort)))))) ;; HTTP request to dest-url for search results - (log-outgoing dest-url) (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5))) (define json (easy:response-json res)) From d2765c2a78a5413d4ab9e7f3e7676521f0372fa6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 2 May 2024 00:01:32 +1200 Subject: [PATCH 10/14] Fix duplicate params->query --- src/data.rkt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/data.rkt b/src/data.rkt index 6975b37..9fd0774 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -57,11 +57,11 @@ (define res (fandom-get-api wikiname - (params->query '(("action" . "query") - ("meta" . "siteinfo") - ("siprop" . "general|rightsinfo") - ("format" . "json") - ("formatversion" . "2"))))) + '(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2")))) (define data (easy:response-json res)) (siteinfo^ (jp "/query/general/sitename" data) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) From 0fd0efc3f2371e25ba8ff5e8725b179428ed7c29 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 4 May 2024 18:01:50 +1200 Subject: [PATCH 11/14] Use default siteinfo when online wiki not found --- src/data.rkt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/data.rkt b/src/data.rkt index 9fd0774..5aba2c2 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -62,11 +62,13 @@ ("siprop" . "general|rightsinfo") ("format" . "json") ("formatversion" . "2")))) - (define data (easy:response-json res)) - (siteinfo^ (jp "/query/general/sitename" data) - (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) - (license^ (jp "/query/rightsinfo/text" data) - (jp "/query/rightsinfo/url" data)))])) + (cond [(= (easy:response-status-code res) 200) + (define data (easy:response-json res)) + (siteinfo^ (jp "/query/general/sitename" data) + (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) + (license^ (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data)))] + [else siteinfo-default])])) (define/memoize (head-data-getter wikiname) #:hash hash ;; data will be stored here, can be referenced by the memoized closure From 2e0bd786ec60591382fbb9f31b9689a03c30b84f Mon Sep 17 00:00:00 2001 From: Evalprime Date: Sun, 25 Feb 2024 20:45:52 +0000 Subject: [PATCH 12/14] add tardis --- src/application-globals.rkt | 2 +- src/extwiki-data.rkt | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index fb8f118..4d8e27e 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -110,7 +110,7 @@ (div (@ (class "niwa__left")) (p ,((extwiki-group^-description group) props)) (p ,((extwiki^-description xt) props)) - (p "This wiki's core community has wholly migrated away from Fandom. You should " + (p "This wiki's core community has largely migrated away from Fandom. You should " (a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!")) (p (@ (class "niwa__feedback")) ,@(add-between diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 5628078..77aa27a 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -71,6 +71,13 @@ (λ (props) '(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop."))) + 'Tardis + (extwiki-group^ + "Tardis" + '(("Forking announcement" . "https://tardis.wiki/wiki/Tardis:Forking_announcement") + ("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/")) + (λ (props) '())) + 'empty (extwiki-group^ "Misc" @@ -419,6 +426,15 @@ (λ (props) `())) + (extwiki^ + '("tardis") 'default + 'Tardis + "Tardis Data Core" + "https://tardis.wiki/wiki/Doctor_Who_Wiki" + "https://tardis.wiki/images/Tardis_images/e/e6/Site-logo.png" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 755efe3cd65968b626c3d661d83c44e468428322 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 5 Jun 2024 23:07:05 +1200 Subject: [PATCH 13/14] Tabber code size and quality --- src/application-globals.rkt | 2 +- static/tabs.js | 92 ++++++++++++------------------------- 2 files changed, 30 insertions(+), 64 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index ff0e81f..e574707 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -172,7 +172,7 @@ (define styles (list (format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies)) - (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin))) + (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.fandom.photoGallery.gallery.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin))) (if (config-true? 'strict_proxy) (map u-proxy-url styles) styles)] diff --git a/static/tabs.js b/static/tabs.js index a077efe..718b48e 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -1,74 +1,40 @@ "use strict"; -let tabToFind = location.hash.length > 1 ? location.hash.substring(1) : null; -for (let tabber of document.body.querySelectorAll(".wds-tabber")) { - let [tabs, contents] = getTabs(tabber); +const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null - for (let i in tabs) { - let tab = tabs[i]; - let content = contents[i]; +for (const tabber of document.body.querySelectorAll(".wds-tabber")) { + for (const [tab, content] of getTabberTabs(tabber)) { + // set up click listener on every tab + tab.addEventListener("click", e => { + setCurrentTab(tabber, tab, content) + e.preventDefault() + }) - tab.addEventListener("click", function(e) { - setCurrentTab(tabber, tab, content); - e.preventDefault(); - }); - if (tab.dataset.hash === tabToFind) { - setCurrentTab(tabber, tab, content); - } - } -} -document.body.classList.remove("bw-tabs-nojs"); - - - -function getTabs(tabber) { - let tabs = []; - let contents = []; - - for (let i of tabber.querySelector(".wds-tabs__wrapper").querySelectorAll(".wds-tabs__tab")) { - tabs.push(i); - } - for (let i of tabber.children) { - if (!i.matches(".wds-tab__content")) { - continue; - } - contents.push(i); - } - - return [tabs, contents]; + // re-open a specific tab on page load based on the URL hash + if (tab.dataset.hash === tabFromHash) { + setCurrentTab(tabber, tab, content) + tab.scrollIntoView() + } + } } -function getCurrentTab(tabber) { - let tab = null; - let content = null; - - tab = tabber.querySelector(".wds-tabs__wrapper").querySelector(".wds-tabs__tab.wds-is-current"); - for (let i of tabber.children) { - if (!i.matches(".wds-tab__content.wds-is-current")) { - continue; - } - content = i; - break; - } - - return [tab, content]; +function getTabberTabs(tabber) { + // need to scope the selector to handle nested tabs. see /unturned/wiki/Crate for an example + const tabs = [...tabber.querySelectorAll(":scope > .wds-tabs__wrapper .wds-tabs__tab")] + const contents = [...tabber.querySelectorAll(":scope > .wds-tab__content")] + return tabs.map((_, index) => [tabs[index], contents[index]]) // transpose arrays into [[tab, content], ...] } function setCurrentTab(tabber, tab, content) { - let [currentTab, currentContent] = getCurrentTab(tabber); - if (currentTab) { - currentTab.classList.remove("wds-is-current"); - } - if (currentContent) { - currentContent.classList.remove("wds-is-current"); - } + // clear currently selected tab + getTabberTabs(tabber).flat().forEach(e => e.classList.remove("wds-is-current")) - tab.classList.add("wds-is-current"); - content.classList.add("wds-is-current"); - if (tab.dataset.hash) { - let fragment = "#" + tab.dataset.hash; - if (location.hash !== fragment) { - history.pushState(null, "", fragment); - } - } + // select new tab + tab.classList.add("wds-is-current") + content.classList.add("wds-is-current") + if (tab.dataset.hash) { + history.replaceState(null, "", `#${tab.dataset.hash}`) + } } + +document.body.classList.remove("bw-tabs-nojs") From 9386d1c93e2b6feddd6e59aa20f5b58c6da3acae Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 6 Nov 2023 20:20:00 +1100 Subject: [PATCH 14/14] Hide slideshows --- static/main.css | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/static/main.css b/static/main.css index bcd612b..f3e65ff 100644 --- a/static/main.css +++ b/static/main.css @@ -218,6 +218,13 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ display: inline !important; } +/* hide slideshows since they're broken + * example: ben10/wiki/Celestialsapien_(Classic)#Powers_and_Abilities + */ +.wikia-slideshow { + display: none; +} + /* animated slots */ #mw-content-text .animated > :not(.animated-active), #mw-content-text .animated > .animated-subframe > :not(.animated-active) { display: inline-block;