From 9773e62c46b3faebb8e621718f3e9870fbd24d10 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 29 May 2023 23:02:28 +0700 Subject: [PATCH 01/27] Add better support for tabs Some pages break without actual tab support, such as https://breezewiki.com/ben10/wiki/Ultimatrix_(Original)#Modes This change aims to work with old browsers (such as Firefox for Android 68) and browsers with Javascript disabled (by showing all tab contents and hiding the tab bar, i.e. how tabs work before this change). --- src/application-globals.rkt | 3 +- static/main.css | 6 ++-- static/tabs.js | 63 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 static/tabs.js diff --git a/src/application-globals.rkt b/src/application-globals.rkt index de60820..a413785 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -203,10 +203,11 @@ `(script (@ (type "module") (src ,(get-static-url "search-suggestions.js")))) "") (script (@ (type "module") (src ,(get-static-url "countdown.js")))) + (script (@ (defer) (src ,(get-static-url "tabs.js")))) (link (@ (rel "icon") (href ,(u (λ (v) (config-true? 'strict_proxy)) (λ (v) (u-proxy-url v)) (head-data^-icon-url head-data)))))) - (body (@ (class ,(head-data^-body-class head-data))) + (body (@ (class ,(head-data^-body-class head-data) " bw-tabs-nojs")) ,(let ([extension-eligible? (cond/var [(not req) #f] diff --git a/static/main.css b/static/main.css index cb32df5..6b07d2b 100644 --- a/static/main.css +++ b/static/main.css @@ -202,11 +202,11 @@ figcaption, .lightbox-caption, .thumbcaption { padding: 0; } -/* show tabs always */ -.wds-tabs__wrapper { +/* show tabs if tabs.js isn't loaded */ +.bw-tabs-nojs .wds-tabs__wrapper { display: none; } -.wds-tab__content { +.bw-tabs-nojs .wds-tab__content { display: block; } diff --git a/static/tabs.js b/static/tabs.js new file mode 100644 index 0000000..589261d --- /dev/null +++ b/static/tabs.js @@ -0,0 +1,63 @@ +"use strict"; + +function handleTabber(tabber) { + let [tabs, contents] = getTabs(tabber); + + for (let i in tabs) { + let tab = tabs[i]; + let content = contents[i]; + tab.addEventListener("click", function(e) { + let [currentTab, currentContent] = getCurrentTab(tabber); + if (currentTab) { + currentTab.classList.remove("wds-is-current"); + } + if (currentContent) { + currentContent.classList.remove("wds-is-current"); + } + + tab.classList.add("wds-is-current"); + content.classList.add("wds-is-current"); + e.preventDefault(); + }); + } +} + +for (let tabber of document.body.querySelectorAll(".wds-tabber")) { + handleTabber(tabber); +} +document.body.classList.remove("bw-tabs-nojs"); + + + +function getTabs(tabber) { + let tabs = []; + let contents = []; + + for (let i of tabber.querySelector(".wds-tabs__wrapper").querySelectorAll(".wds-tabs__tab")) { + tabs.push(i); + } + for (let i of tabber.children) { + if (!i.matches(".wds-tab__content")) { + continue; + } + contents.push(i); + } + + return [tabs, contents]; +} + +function getCurrentTab(tabber) { + let tab = null; + let content = null; + + tab = tabber.querySelector(".wds-tabs__wrapper").querySelector(".wds-tabs__tab.wds-is-current"); + for (let i of tabber.children) { + if (!i.matches(".wds-tab__content.wds-is-current")) { + continue; + } + content = i; + break; + } + + return [tab, content]; +} From ead6896818be96048138d0d3ecd12438f8d475b8 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 30 May 2023 13:41:49 +0700 Subject: [PATCH 02/27] Add the ability to specify/open the last open tab in the URL --- static/tabs.js | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/static/tabs.js b/static/tabs.js index 589261d..916cb4d 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -1,30 +1,23 @@ "use strict"; -function handleTabber(tabber) { +let tabToFind = location.hash.length > 1 ? location.hash.substring(1) : null; +for (let tabber of document.body.querySelectorAll(".wds-tabber")) { let [tabs, contents] = getTabs(tabber); for (let i in tabs) { let tab = tabs[i]; let content = contents[i]; - tab.addEventListener("click", function(e) { - let [currentTab, currentContent] = getCurrentTab(tabber); - if (currentTab) { - currentTab.classList.remove("wds-is-current"); - } - if (currentContent) { - currentContent.classList.remove("wds-is-current"); - } - tab.classList.add("wds-is-current"); - content.classList.add("wds-is-current"); + tab.addEventListener("click", function(e) { + setCurrentTab(tabber, tab, content); e.preventDefault(); }); + if (tab.dataset.hash === tabToFind) { + setCurrentTab(tabber, tab, content); + tabToFind = null; + } } } - -for (let tabber of document.body.querySelectorAll(".wds-tabber")) { - handleTabber(tabber); -} document.body.classList.remove("bw-tabs-nojs"); @@ -61,3 +54,18 @@ function getCurrentTab(tabber) { return [tab, content]; } + +function setCurrentTab(tabber, tab, content) { + let [currentTab, currentContent] = getCurrentTab(tabber); + if (currentTab) { + currentTab.classList.remove("wds-is-current"); + } + if (currentContent) { + currentContent.classList.remove("wds-is-current"); + } + + tab.classList.add("wds-is-current"); + content.classList.add("wds-is-current"); + location.hash = "#" + tab.dataset.hash; + history.pushState(null, "", "#" + tab.dataset.hash); +} From f5399524b135eedf346ad758a88f37cba57be700 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 30 May 2023 14:01:14 +0700 Subject: [PATCH 03/27] Prevent linking to tabs with no IDs --- static/tabs.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/static/tabs.js b/static/tabs.js index 916cb4d..6f582e8 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -14,7 +14,6 @@ for (let tabber of document.body.querySelectorAll(".wds-tabber")) { }); if (tab.dataset.hash === tabToFind) { setCurrentTab(tabber, tab, content); - tabToFind = null; } } } @@ -66,6 +65,8 @@ function setCurrentTab(tabber, tab, content) { tab.classList.add("wds-is-current"); content.classList.add("wds-is-current"); - location.hash = "#" + tab.dataset.hash; - history.pushState(null, "", "#" + tab.dataset.hash); + if (tab.dataset.hash) { + location.hash = "#" + tab.dataset.hash; + history.pushState(null, "", "#" + tab.dataset.hash); + } } From dcb8a8a590df66023cc355266cc1fc0c4f278438 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 6 Nov 2023 20:31:20 +1100 Subject: [PATCH 04/27] Prevent making duplicate history entries --- static/tabs.js | 1 - 1 file changed, 1 deletion(-) diff --git a/static/tabs.js b/static/tabs.js index 6f582e8..4c4cdda 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -67,6 +67,5 @@ function setCurrentTab(tabber, tab, content) { content.classList.add("wds-is-current"); if (tab.dataset.hash) { location.hash = "#" + tab.dataset.hash; - history.pushState(null, "", "#" + tab.dataset.hash); } } From 2b3a8fe1084abab995f6d0937f673dcb4eac78b6 Mon Sep 17 00:00:00 2001 From: blankie Date: Mon, 13 Nov 2023 14:35:07 +1100 Subject: [PATCH 05/27] Fix scrolling to sections if a tab's hash coincides with one ben10/wiki/Alien_X_(Classic)#Appearances --- static/tabs.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/static/tabs.js b/static/tabs.js index 4c4cdda..a077efe 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -66,6 +66,9 @@ function setCurrentTab(tabber, tab, content) { tab.classList.add("wds-is-current"); content.classList.add("wds-is-current"); if (tab.dataset.hash) { - location.hash = "#" + tab.dataset.hash; + let fragment = "#" + tab.dataset.hash; + if (location.hash !== fragment) { + history.pushState(null, "", fragment); + } } } From 86c99906c64268f9c97da751f87f9dee6704591c Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 31 Dec 2023 21:55:56 +1100 Subject: [PATCH 06/27] Fix error on a redirect page with no link https://lists.sr.ht/~cadence/breezewiki-discuss/%3CCY2G0E3G55N3.ANW2QREUS5SO%40nixnetmail.com%3E --- src/page-wiki-offline.rkt | 9 ++++++--- src/page-wiki.rkt | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/page-wiki-offline.rkt b/src/page-wiki-offline.rkt index 1d0323b..3f818ee 100644 --- a/src/page-wiki-offline.rkt +++ b/src/page-wiki-offline.rkt @@ -98,16 +98,19 @@ #:head-data head-data #:siteinfo (siteinfo-fetch wikiname) )) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) (define headers (build-headers always-headers ; redirect-query-parameter: only the string "no" is significant: ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg + (when (and redirect-msg-a (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) (header #"Refresh" value))))) (when (config-true? 'debug) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 8df701c..ae060d4 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -80,16 +80,19 @@ #:title title #:head-data head-data #:siteinfo siteinfo)) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) (define headers (build-headers always-headers ; redirect-query-parameter: only the string "no" is significant: ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg + (when (and redirect-msg-a (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) (header #"Refresh" value))))) (when (config-true? 'debug) From 39423504689a0f7a0dfb8da22258e03072d1552d Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 31 Dec 2023 21:55:56 +1100 Subject: [PATCH 07/27] Fix error on a redirect page with no link https://lists.sr.ht/~cadence/breezewiki-discuss/%3CCY2G0E3G55N3.ANW2QREUS5SO%40nixnetmail.com%3E --- src/page-wiki-offline.rkt | 9 ++++++--- src/page-wiki.rkt | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/page-wiki-offline.rkt b/src/page-wiki-offline.rkt index 1d0323b..3f818ee 100644 --- a/src/page-wiki-offline.rkt +++ b/src/page-wiki-offline.rkt @@ -98,16 +98,19 @@ #:head-data head-data #:siteinfo (siteinfo-fetch wikiname) )) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) (define headers (build-headers always-headers ; redirect-query-parameter: only the string "no" is significant: ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg + (when (and redirect-msg-a (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) (header #"Refresh" value))))) (when (config-true? 'debug) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index 8df701c..ae060d4 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -80,16 +80,19 @@ #:title title #:head-data head-data #:siteinfo siteinfo)) - (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes")) + (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) + (define redirect-msg-a (if redirect-msg + ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg)) + #f)) (define headers (build-headers always-headers ; redirect-query-parameter: only the string "no" is significant: ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367 - (when (and redirect-msg + (when (and redirect-msg-a (not (equal? redirect-query-parameter "no"))) - (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] + (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) (header #"Refresh" value))))) (when (config-true? 'debug) From 0fed724604b95cdf2e0227f3e788aa341b30b387 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 9 Jan 2024 17:13:04 +1300 Subject: [PATCH 08/27] Fix redirects to category pages in offline mode --- src/page-wiki-offline.rkt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/page-wiki-offline.rkt b/src/page-wiki-offline.rkt index 3f818ee..2b52373 100644 --- a/src/page-wiki-offline.rkt +++ b/src/page-wiki-offline.rkt @@ -126,7 +126,8 @@ ;; page not found on disk, perhaps it's a redirect? redirects are stored in the database (var target (query-maybe-value* "select redirect from page where wikiname = ? and basename = ?" wikiname basename)) [target - (generate-redirect (basename->name-for-query target))] + ; don't url decode the target, or Category: pages will be interpreted as a protocol + (generate-redirect (regexp-replace* #rx"#" target "/"))] ;; breezewiki doesn't have the page archived, see if we can make a network request for it [(not (config-true? 'feature_offline::only)) From b8a6c5198d1a4d6e4907e1346033a054912fcfcf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 9 Jan 2024 17:35:26 +1300 Subject: [PATCH 09/27] Fix more redirects in offline mode --- src/page-wiki-offline.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-wiki-offline.rkt b/src/page-wiki-offline.rkt index 2b52373..906718d 100644 --- a/src/page-wiki-offline.rkt +++ b/src/page-wiki-offline.rkt @@ -127,7 +127,7 @@ (var target (query-maybe-value* "select redirect from page where wikiname = ? and basename = ?" wikiname basename)) [target ; don't url decode the target, or Category: pages will be interpreted as a protocol - (generate-redirect (regexp-replace* #rx"#" target "/"))] + (generate-redirect (format "/~a/wiki/~a" wikiname (regexp-replace* #rx"#" target "/")))] ;; breezewiki doesn't have the page archived, see if we can make a network request for it [(not (config-true? 'feature_offline::only)) From b02e2a405329ec56c5ce8fd357fd60ff11332033 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 11:21:42 +1300 Subject: [PATCH 10/27] Fix failing test after search was refactored --- src/page-search.rkt | 4 ++-- src/search-provider-fandom.rkt | 14 ++++++++------ src/search-provider-solr.rkt | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/page-search.rkt b/src/page-search.rkt index 193dbd6..019ebfe 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -25,8 +25,8 @@ page-search) (define search-providers - (hash "fandom" generate-results-content-fandom - "solr" generate-results-content-solr)) + (hash "fandom" search-fandom + "solr" search-solr)) ;; this takes the info we gathered from fandom and makes the big fat x-expression page (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f]) diff --git a/src/search-provider-fandom.rkt b/src/search-provider-fandom.rkt index 945b111..2338c13 100644 --- a/src/search-provider-fandom.rkt +++ b/src/search-provider-fandom.rkt @@ -8,15 +8,15 @@ "../lib/xexpr-utils.rkt") (provide - generate-results-content-fandom) + search-fandom) (module+ test (require rackunit "test-utils.rkt") - (define search-json-data - '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) + (define search-results-data + '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) -(define (generate-results-content-fandom wikiname query params) +(define (search-fandom wikiname query params) ;; constructing the URL where I want to get fandom data from... (define origin (format "https://~a.fandom.com" wikiname)) ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json @@ -33,8 +33,10 @@ (define res (easy:get dest-url #:timeouts timeouts)) (define json (easy:response-json res)) (define search-results (jp "/query/search" json)) + (generate-results-content-fandom wikiname query search-results)) - ;; generate content for display in the wiki page layout +;;; generate content for display in the wiki page layout +(define (generate-results-content-fandom wikiname query search-results) `(div (@ (class "mw-parser-output")) ;; header before the search results showing how many we found (p ,(format "~a results found for " (length search-results)) @@ -60,4 +62,4 @@ (module+ test (parameterize ([(config-parameter 'feature_offline::only) "false"]) (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") - (generate-results-content-fandom test-req "" "test" "Gacha" search-json-data)))))) + (generate-results-content-fandom "test" "Gacha" search-results-data)))))) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index ed9fb7b..1ec48e2 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -9,11 +9,11 @@ "../lib/xexpr-utils.rkt") (provide - generate-results-content-solr) + search-solr) (struct result^ (hl-title hl-body kb words page-path) #:transparent) -(define (generate-results-content-solr wikiname query params) +(define (search-solr wikiname query params) ;; grab things from params that would modify the search (define op (if (equal? (dict-ref params 'op #f) "or") '("or" . "OR") '("and" . "AND"))) (define sort (if (equal? (dict-ref params 'sort #f) "len") '("len" . "len desc") '("relevance" . "score desc"))) From a52d131b936037fdd7847bf8d58c1662227101ad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 11 Jan 2024 22:33:59 +1300 Subject: [PATCH 11/27] Split massive uploads in Solr indexer --- archiver/fts.rkt | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/archiver/fts.rkt b/archiver/fts.rkt index c2f597b..6a00041 100644 --- a/archiver/fts.rkt +++ b/archiver/fts.rkt @@ -8,6 +8,7 @@ racket/promise racket/port racket/runtime-path + racket/sequence racket/string file/gunzip db @@ -156,8 +157,16 @@ (define data (cond [(and (read-from-cache?) (file-exists? "cache.rkt")) - (displayln "Reading in...") - (with-input-from-file "cache.rkt" (λ () (read)))] + (define size (file-size "cache.rkt")) + (call-with-input-file "cache.rkt" + (λ (in) + (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position in) 64 1024)) + (ceiling (/ size 64 1024)) + "Reading in...")) + 2)) + (begin0 + (read in) + (quit))))] [else (define x (box (progress^ 0 1 "..."))) (define quit (make-progress (λ () (unbox x)))) @@ -183,18 +192,22 @@ (display "Converting... ") (flush-output) - (define ser (jsexpr->bytes data)) - (define ser-port (open-input-bytes ser)) - (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) - (ceiling (/ (bytes-length ser) 64 1024)) - "Posting...")) - 2)) - (define res - (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) - #:data ser-port - #:headers '#hasheq((Content-Type . "application/json")) - #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) - (quit) - (displayln (response-status-line res))) + (define slice-size 30000) + (define slices (ceiling (/ (length data) slice-size))) + (for ([slice (in-slice slice-size data)] + [i (in-naturals 1)]) + (define ser (jsexpr->bytes slice)) + (define ser-port (open-input-bytes ser)) + (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) + (ceiling (/ (bytes-length ser) 64 1024)) + (format "Posting... (~a/~a)" i slices))) + 2)) + (define res + (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) + #:data ser-port + #:headers '#hasheq((Content-Type . "application/json")) + #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) + (quit) + (displayln (response-status-line res)))) (run start) From 6260ba809bdd2321fa59c6133f749f2db0d23e17 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 1 May 2024 00:53:09 +1200 Subject: [PATCH 12/27] Fix running out of file descriptors --- lib/thread-utils.rkt | 3 +-- src/application-globals.rkt | 3 --- src/data.rkt | 20 +++++++------- src/dispatcher-tree.rkt | 21 +++++++++++++-- src/fandom-request.rkt | 48 ++++++++++++++++++++++++++++++++++ src/page-category.rkt | 42 +++++++++++++---------------- src/page-file.rkt | 23 +++++++--------- src/page-search.rkt | 2 -- src/page-wiki.rkt | 34 +++++++++++++----------- src/search-provider-fandom.rkt | 24 +++++++---------- src/search-provider-solr.rkt | 3 +-- src/whole-utils.rkt | 11 -------- 12 files changed, 134 insertions(+), 100 deletions(-) create mode 100644 src/fandom-request.rkt delete mode 100644 src/whole-utils.rkt diff --git a/lib/thread-utils.rkt b/lib/thread-utils.rkt index f907dac..66e2b4c 100644 --- a/lib/thread-utils.rkt +++ b/lib/thread-utils.rkt @@ -1,6 +1,5 @@ #lang racket/base -(require (prefix-in easy: net/http-easy) - "../src/data.rkt" +(require "../src/data.rkt" "xexpr-utils.rkt") (provide diff --git a/src/application-globals.rkt b/src/application-globals.rkt index de60820..fb8f118 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -22,8 +22,6 @@ (provide ; headers to always send on all http responses always-headers - ; timeout durations for http-easy requests - timeouts ; generates a consistent footer application-footer ; generates a consistent template for wiki page content to sit in @@ -39,7 +37,6 @@ (define always-headers (list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom (header #"Link" (string->bytes/latin-1 link-header)))) -(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define-runtime-path path-static "../static") (define theme-icons diff --git a/src/data.rkt b/src/data.rkt index b22e8a0..6975b37 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -7,8 +7,8 @@ (prefix-in easy: net/http-easy) db memo + "fandom-request.rkt" "static-data.rkt" - "whole-utils.rkt" "../lib/url-utils.rkt" "../lib/xexpr-utils.rkt" "../archiver/archiver-database.rkt" @@ -54,16 +54,14 @@ (vector-ref row 3))) siteinfo-default)] [else - (define dest-url - (format "https://~a.fandom.com/api.php?~a" - wikiname - (params->query '(("action" . "query") - ("meta" . "siteinfo") - ("siprop" . "general|rightsinfo") - ("format" . "json") - ("formatversion" . "2"))))) - (log-outgoing dest-url) - (define res (easy:get dest-url)) + (define res + (fandom-get-api + wikiname + (params->query '(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2"))))) (define data (easy:response-json res)) (siteinfo^ (jp "/query/general/sitename" data) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 48e8ebb..0212242 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -33,12 +33,29 @@ ; don't forget that I'm returning *code* - return a call to the function (datum->syntax stx `(make-dispatcher-tree ,ds))) +; guard that the page returned a response, otherwise print more detailed debugging information +(define-syntax-rule (page ds name) + (λ (req) + (define dispatcher (hash-ref ds (quote name))) + (define page-response (dispatcher req)) + (if (response? page-response) + page-response + (response/output + #:code 500 + #:mime-type #"text/plain" + (λ (out) + (for ([port (list (current-error-port) out)]) + (parameterize ([current-output-port port]) + (printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n" + (quote name) + page-response)))))))) + (define (make-dispatcher-tree ds) (define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher)) (define tree (sequencer:make subdomain-dispatcher - (pathprocedure:make "/" (hash-ref ds 'page-home)) + (pathprocedure:make "/" (page ds page-home)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) @@ -48,7 +65,7 @@ (if (config-true? 'feature_offline::enabled) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) - (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (if (config-true? 'feature_offline::enabled) diff --git a/src/fandom-request.rkt b/src/fandom-request.rkt new file mode 100644 index 0000000..966eeee --- /dev/null +++ b/src/fandom-request.rkt @@ -0,0 +1,48 @@ +#lang typed/racket/base +(require "config.rkt" + "../lib/url-utils.rkt") +(define-type Headers (HashTable Symbol (U Bytes String))) +(require/typed net/http-easy + [#:opaque Timeout-Config timeout-config?] + [#:opaque Response response?] + [#:opaque Session session?] + [current-session (Parameter Session)] + [make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)] + [get ((U Bytes String) + [#:close? Boolean] + [#:headers Headers] + [#:timeouts Timeout-Config] + [#:max-attempts Exact-Positive-Integer] + [#:max-redirects Exact-Nonnegative-Integer] + [#:user-agent (U Bytes String)] + -> Response)]) + +(provide + fandom-get + fandom-get-api + timeouts) + +(define timeouts (make-timeout-config #:lease 5 #:connect 5)) + +(: no-headers Headers) +(define no-headers '#hasheq()) + +(: fandom-get (String String [#:headers (Option Headers)] -> Response)) +(define (fandom-get wikiname path #:headers [headers #f]) + (define dest-url (string-append "https://www.fandom.com" path)) + (define host (string-append wikiname ".fandom.com")) + (log-outgoing wikiname path) + (get dest-url + #:timeouts timeouts + #:headers (hash-set (or headers no-headers) 'Host host))) + +(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response)) +(define (fandom-get-api wikiname params #:headers [headers #f]) + (fandom-get wikiname + (string-append "/api.php?" (params->query params)) + #:headers headers)) + +(: log-outgoing (String String -> Void)) +(define (log-outgoing wikiname path) + (when (config-true? 'log_outgoing) + (printf "out: ~a ~a~n" wikiname path))) diff --git a/src/page-category.rkt b/src/page-category.rkt index 213d423..e1fe659 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -15,11 +15,11 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "page-wiki.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -73,30 +73,24 @@ (define-values (members-data page-data siteinfo) (thread-values (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) + (easy:response-json + (fandom-get-api + wikiname + `(("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))))) (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,prefixed-category) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) + (easy:response-json + (fandom-get-api + wikiname + `(("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))))) (λ () (siteinfo-fetch wikiname)))) diff --git a/src/page-file.rkt b/src/page-file.rkt index 2a7332c..5151f1d 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -15,11 +15,11 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "page-wiki.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide page-file) @@ -40,8 +40,7 @@ (imageDescription . #f)))) (define (url-content-type url) - (log-outgoing url) - (define dest-res (easy:head url #:timeouts timeouts)) + (define dest-res (easy:head url)) (easy:response-headers-ref dest-res 'content-type)) (define (get-media-html url content-type) @@ -106,20 +105,18 @@ (response-handler (define wikiname (path/param-path (first (url-path (request-uri req))))) (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-title)) + (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title)) (define-values (media-detail siteinfo) (thread-values (λ () - (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . ,prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) + (define dest-res + (fandom-get + wikiname + (format "/wikia.php?~a" + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title)))))) (easy:response-json dest-res)) (λ () (siteinfo-fetch wikiname)))) diff --git a/src/page-search.rkt b/src/page-search.rkt index 019ebfe..39f361a 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -2,7 +2,6 @@ (require racket/dict racket/list racket/string - (prefix-in easy: net/http-easy) ; html libs html-writing ; web server libs @@ -18,7 +17,6 @@ "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index ae060d4..f16792c 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -17,12 +17,12 @@ "application-globals.rkt" "config.rkt" "data.rkt" + "fandom-request.rkt" "../lib/pure-utils.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" "../lib/tree-updater.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -38,25 +38,20 @@ (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) (define user-cookies (user-cookies-getter req)) - (define origin (format "https://~a.fandom.com" wikiname)) (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define-values (dest-res siteinfo) (thread-values (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,path) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (easy:get dest-url - #:timeouts timeouts - #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) + (fandom-get-api + wikiname + `(("action" . "parse") + ("page" . ,path) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json")) + #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) (λ () (siteinfo-fetch wikiname)))) @@ -103,4 +98,13 @@ #:code 200 #:headers headers (λ (out) - (write-html body out))))))])) + (write-html body out))))))] + [(eq? 404 (easy:response-status-code dest-res)) + (next-dispatcher)] + [else + (response-handler + (error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" + wikiname + path + (easy:response-status-code dest-res) + (easy:response-body dest-res)))])) diff --git a/src/search-provider-fandom.rkt b/src/search-provider-fandom.rkt index 2338c13..b8dd48f 100644 --- a/src/search-provider-fandom.rkt +++ b/src/search-provider-fandom.rkt @@ -3,8 +3,8 @@ (prefix-in easy: net/http-easy) "application-globals.rkt" "config.rkt" + "fandom-request.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -17,20 +17,14 @@ '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) (define (search-fandom wikiname query params) - ;; constructing the URL where I want to get fandom data from... - (define origin (format "https://~a.fandom.com" wikiname)) - ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "search") - ("srsearch" . ,query) - ("formatversion" . "2") - ("format" . "json"))))) - ;; HTTP request to dest-url for search results - (log-outgoing dest-url) - (define res (easy:get dest-url #:timeouts timeouts)) + (define res + (fandom-get-api + wikiname + `(("action" . "query") + ("list" . "search") + ("srsearch" . ,query) + ("formatversion" . "2") + ("format" . "json")))) (define json (easy:response-json res)) (define search-results (jp "/query/search" json)) (generate-results-content-fandom wikiname query search-results)) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index 1ec48e2..31813da 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -5,7 +5,6 @@ "application-globals.rkt" "../lib/html-parsing/main.rkt" "../lib/url-utils.rkt" - "whole-utils.rkt" "../lib/xexpr-utils.rkt") (provide @@ -37,7 +36,7 @@ ("sort" . ,(cdr sort)))))) ;; HTTP request to dest-url for search results (log-outgoing dest-url) - (define res (easy:get dest-url #:timeouts timeouts)) + (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5))) (define json (easy:response-json res)) ;; build result objects diff --git a/src/whole-utils.rkt b/src/whole-utils.rkt deleted file mode 100644 index 7118866..0000000 --- a/src/whole-utils.rkt +++ /dev/null @@ -1,11 +0,0 @@ -#lang typed/racket/base -(require "config.rkt") - -(provide - ; prints "out: " - log-outgoing) - -(: log-outgoing (String -> Void)) -(define (log-outgoing url-string) - (when (config-true? 'log_outgoing) - (printf "out: ~a~n" url-string))) From 7dff049ece876d1d31d4537c2176629cabdd7af0 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 1 May 2024 00:57:13 +1200 Subject: [PATCH 13/27] Wrap all pages in response safety checker --- src/dispatcher-tree.rkt | 20 ++++++++++---------- src/search-provider-solr.rkt | 1 - 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index 0212242..a967095 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -56,20 +56,20 @@ (sequencer:make subdomain-dispatcher (pathprocedure:make "/" (page ds page-home)) - (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) - (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) - (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) - (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works)) - (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) - (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) + (pathprocedure:make "/proxy" (page ds page-proxy)) + (pathprocedure:make "/search" (page ds page-global-search)) + (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings)) + (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works)) + (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category))) + (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file))) (if (config-true? 'feature_offline::enabled) - (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) + (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline))) (λ (_conn _req) (next-dispatcher))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki))) - (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) - (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) + (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search))) + (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home))) (if (config-true? 'feature_offline::enabled) - (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive))) + (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive))) (λ (_conn _req) (next-dispatcher))) (hash-ref ds 'static-dispatcher) (lift:make (hash-ref ds 'page-not-found)))) diff --git a/src/search-provider-solr.rkt b/src/search-provider-solr.rkt index 31813da..c15e31f 100644 --- a/src/search-provider-solr.rkt +++ b/src/search-provider-solr.rkt @@ -35,7 +35,6 @@ ("hl.tag.post" . "") ("sort" . ,(cdr sort)))))) ;; HTTP request to dest-url for search results - (log-outgoing dest-url) (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5))) (define json (easy:response-json res)) From d2765c2a78a5413d4ab9e7f3e7676521f0372fa6 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 2 May 2024 00:01:32 +1200 Subject: [PATCH 14/27] Fix duplicate params->query --- src/data.rkt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/data.rkt b/src/data.rkt index 6975b37..9fd0774 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -57,11 +57,11 @@ (define res (fandom-get-api wikiname - (params->query '(("action" . "query") - ("meta" . "siteinfo") - ("siprop" . "general|rightsinfo") - ("format" . "json") - ("formatversion" . "2"))))) + '(("action" . "query") + ("meta" . "siteinfo") + ("siprop" . "general|rightsinfo") + ("format" . "json") + ("formatversion" . "2")))) (define data (easy:response-json res)) (siteinfo^ (jp "/query/general/sitename" data) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) From 0fd0efc3f2371e25ba8ff5e8725b179428ed7c29 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 4 May 2024 18:01:50 +1200 Subject: [PATCH 15/27] Use default siteinfo when online wiki not found --- src/data.rkt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/data.rkt b/src/data.rkt index 9fd0774..5aba2c2 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -62,11 +62,13 @@ ("siprop" . "general|rightsinfo") ("format" . "json") ("formatversion" . "2")))) - (define data (easy:response-json res)) - (siteinfo^ (jp "/query/general/sitename" data) - (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) - (license^ (jp "/query/rightsinfo/text" data) - (jp "/query/rightsinfo/url" data)))])) + (cond [(= (easy:response-status-code res) 200) + (define data (easy:response-json res)) + (siteinfo^ (jp "/query/general/sitename" data) + (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) + (license^ (jp "/query/rightsinfo/text" data) + (jp "/query/rightsinfo/url" data)))] + [else siteinfo-default])])) (define/memoize (head-data-getter wikiname) #:hash hash ;; data will be stored here, can be referenced by the memoized closure From 2e0bd786ec60591382fbb9f31b9689a03c30b84f Mon Sep 17 00:00:00 2001 From: Evalprime Date: Sun, 25 Feb 2024 20:45:52 +0000 Subject: [PATCH 16/27] add tardis --- src/application-globals.rkt | 2 +- src/extwiki-data.rkt | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index fb8f118..4d8e27e 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -110,7 +110,7 @@ (div (@ (class "niwa__left")) (p ,((extwiki-group^-description group) props)) (p ,((extwiki^-description xt) props)) - (p "This wiki's core community has wholly migrated away from Fandom. You should " + (p "This wiki's core community has largely migrated away from Fandom. You should " (a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!")) (p (@ (class "niwa__feedback")) ,@(add-between diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 5628078..77aa27a 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -71,6 +71,13 @@ (λ (props) '(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop."))) + 'Tardis + (extwiki-group^ + "Tardis" + '(("Forking announcement" . "https://tardis.wiki/wiki/Tardis:Forking_announcement") + ("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/")) + (λ (props) '())) + 'empty (extwiki-group^ "Misc" @@ -419,6 +426,15 @@ (λ (props) `())) + (extwiki^ + '("tardis") 'default + 'Tardis + "Tardis Data Core" + "https://tardis.wiki/wiki/Doctor_Who_Wiki" + "https://tardis.wiki/images/Tardis_images/e/e6/Site-logo.png" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 755efe3cd65968b626c3d661d83c44e468428322 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 5 Jun 2024 23:07:05 +1200 Subject: [PATCH 17/27] Tabber code size and quality --- src/application-globals.rkt | 2 +- static/tabs.js | 92 ++++++++++++------------------------- 2 files changed, 30 insertions(+), 64 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index ff0e81f..e574707 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -172,7 +172,7 @@ (define styles (list (format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies)) - (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin))) + (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.fandom.photoGallery.gallery.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin))) (if (config-true? 'strict_proxy) (map u-proxy-url styles) styles)] diff --git a/static/tabs.js b/static/tabs.js index a077efe..718b48e 100644 --- a/static/tabs.js +++ b/static/tabs.js @@ -1,74 +1,40 @@ "use strict"; -let tabToFind = location.hash.length > 1 ? location.hash.substring(1) : null; -for (let tabber of document.body.querySelectorAll(".wds-tabber")) { - let [tabs, contents] = getTabs(tabber); +const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null - for (let i in tabs) { - let tab = tabs[i]; - let content = contents[i]; +for (const tabber of document.body.querySelectorAll(".wds-tabber")) { + for (const [tab, content] of getTabberTabs(tabber)) { + // set up click listener on every tab + tab.addEventListener("click", e => { + setCurrentTab(tabber, tab, content) + e.preventDefault() + }) - tab.addEventListener("click", function(e) { - setCurrentTab(tabber, tab, content); - e.preventDefault(); - }); - if (tab.dataset.hash === tabToFind) { - setCurrentTab(tabber, tab, content); - } - } -} -document.body.classList.remove("bw-tabs-nojs"); - - - -function getTabs(tabber) { - let tabs = []; - let contents = []; - - for (let i of tabber.querySelector(".wds-tabs__wrapper").querySelectorAll(".wds-tabs__tab")) { - tabs.push(i); - } - for (let i of tabber.children) { - if (!i.matches(".wds-tab__content")) { - continue; - } - contents.push(i); - } - - return [tabs, contents]; + // re-open a specific tab on page load based on the URL hash + if (tab.dataset.hash === tabFromHash) { + setCurrentTab(tabber, tab, content) + tab.scrollIntoView() + } + } } -function getCurrentTab(tabber) { - let tab = null; - let content = null; - - tab = tabber.querySelector(".wds-tabs__wrapper").querySelector(".wds-tabs__tab.wds-is-current"); - for (let i of tabber.children) { - if (!i.matches(".wds-tab__content.wds-is-current")) { - continue; - } - content = i; - break; - } - - return [tab, content]; +function getTabberTabs(tabber) { + // need to scope the selector to handle nested tabs. see /unturned/wiki/Crate for an example + const tabs = [...tabber.querySelectorAll(":scope > .wds-tabs__wrapper .wds-tabs__tab")] + const contents = [...tabber.querySelectorAll(":scope > .wds-tab__content")] + return tabs.map((_, index) => [tabs[index], contents[index]]) // transpose arrays into [[tab, content], ...] } function setCurrentTab(tabber, tab, content) { - let [currentTab, currentContent] = getCurrentTab(tabber); - if (currentTab) { - currentTab.classList.remove("wds-is-current"); - } - if (currentContent) { - currentContent.classList.remove("wds-is-current"); - } + // clear currently selected tab + getTabberTabs(tabber).flat().forEach(e => e.classList.remove("wds-is-current")) - tab.classList.add("wds-is-current"); - content.classList.add("wds-is-current"); - if (tab.dataset.hash) { - let fragment = "#" + tab.dataset.hash; - if (location.hash !== fragment) { - history.pushState(null, "", fragment); - } - } + // select new tab + tab.classList.add("wds-is-current") + content.classList.add("wds-is-current") + if (tab.dataset.hash) { + history.replaceState(null, "", `#${tab.dataset.hash}`) + } } + +document.body.classList.remove("bw-tabs-nojs") From 5672f468862951837f54c792bddd9758f64a4738 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Jul 2024 01:28:27 +1200 Subject: [PATCH 18/27] Add new independent wikis --- src/application-globals.rkt | 2 +- src/extwiki-data.rkt | 68 +++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index e574707..6d940ac 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -108,8 +108,8 @@ (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,(extwiki^-name xt) " →") (div (@ (class "niwa__cols")) (div (@ (class "niwa__left")) - (p ,((extwiki-group^-description group) props)) (p ,((extwiki^-description xt) props)) + (p ,((extwiki-group^-description group) props)) (p "This wiki's core community has largely migrated away from Fandom. You should " (a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!")) (p (@ (class "niwa__feedback")) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 77aa27a..29997c4 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -27,6 +27,13 @@ (λ (props) `(p "The Square Enix Indpendent Wiki Alliance, or SEIWA, is a network of independent wikis established in 2011 and focused on providing high-quality coverage of Square Enix and its content. We work together, along with our affiliates and others, to co-operate and support one another while providing the best-quality content on the various Square Enix video games and media."))) + 'GWN + (extwiki-group^ + "GWN" + '(("Gaming Wiki Network" . "https://gamingwikinetwork.org/")) + (λ (props) + `(p "This wiki is part of the Gaming Wiki Network, a network of independently-hosted wikis about video game franchises. The GWN was founded on October 21, 2022. It aims to support all gaming communities in building independently-hosted wikis."))) + 'Terraria (extwiki-group^ "Terraria" @@ -316,11 +323,11 @@ (extwiki^ '("zelda" "zelda-archive") 'default 'NIWA - "Zeldapedia" - "https://zeldapedia.wiki/wiki/Main_Page" + "Zelda Wiki" + "https://zeldawiki.wiki/wiki/Main_Page" "https://niwanetwork.org/images/logos/zeldapedia.png" (λ (props) - `((p "Founded on April 23, 2005 as Zelda Wiki, today's Zeldapedia is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zeldapedia went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons.")))) + `((p "Founded on April 23, 2005, Zelda Wiki is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zelda Wiki went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons.")))) (extwiki^ '("chrono") 'default @@ -435,6 +442,61 @@ (λ (props) `())) + (extwiki^ + '("wizardry") 'default + 'GWN + "Wizardry Wiki" + "https://wizardry.wiki.gg/wiki/Wizardry_Wiki" + "https://wizardry.wiki.gg/images/e/e6/Site-logo.png" + (λ (props) + `((p "On March 21, 2023, the wiki has decided to leave and abandoning from Fandom due to numerous of issues such as intrusive advertising, long-lasting bugs, restrictions on customization, etcetera. Wizardry Wiki was officially inducted into the wiki.gg wikifarm, with all contents forked over.") + (p "The wiki has partnered with " (a (@ (href "https://fallout.wiki/")) "Independent Fallout Wiki") " as of June 14, 2024.")))) + + (extwiki^ + '("jackryan") 'default + 'GWN + "Tom Clancy Wiki" + "https://tomclancy.wiki.gg/wiki/Tom_Clancy_Wiki" + "https://tomclancy.wiki.gg/images/thumb/c/c5/Jack_Ryan_Logo_Dark.png/600px-Jack_Ryan_Logo_Dark.png" + (λ (props) + `((p "The Tom Clancy Wiki is a collaborative encyclopedia dedicated to Tom Clancy’s franchises. The Tom Clancy franchise is a 40-year old expansive franchise founded by Tom Clancy, telling several unique sagas through books, video games, and films, as well as a TV show.")))) + + (extwiki^ + '("hollowknight") 'default + 'GWN + "Hollow Knight Wiki" + "https://hollowknight.wiki/wiki/Main_Page" + "https://gamingwikinetwork.org/images/logos/hollowknight.png" + (λ (props) + `((p "We are an independently hosted wiki for the games Hollow Knight and Hollow Knight: Silksong, created by fans, for fans. The wiki is a fork of the FANDOM Hollow Knight Wiki and was officially unveiled on October 31, 2023.")))) + + (extwiki^ + '("hellokitty" "sanrio") 'default + 'GWN + "Sanrio Wiki" + "https://sanriowiki.com/wiki/Sanrio_Wiki" + "https://cdn.sanriowiki.com/wiki.png" + (λ (props) + `((p "Sanrio Wiki is a project that was started on April 14, 2015 by EvieMelody. It was hosted on the wiki-farm ShoutWiki and has since become independent.")))) + + (extwiki^ + '("sto") 'default + 'GWN + "Star Trek Online Wiki" + "https://stowiki.net/wiki/Main_Page" + "https://gamingwikinetwork.org/images/logos/stowiki.png" + (λ (props) + `())) + + (extwiki^ + '("rayman-game" "ubisoftrayman") 'default + 'GWN + "Rayman Wiki" + "https://raymanpc.com/wiki/en/Main_Page" + "https://raymanpc.com/wiki/script-en/resources/assets/logo-en.png?5c608" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 1ef184547b2be6715318c3d749d3114fe61866ec Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Jul 2024 02:28:17 +1200 Subject: [PATCH 19/27] Allow minimising independent wiki notice --- src/application-globals.rkt | 18 ++++++++++-------- src/data.rkt | 25 ++++++++++++++++++++----- static/main.css | 17 +++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index 6d940ac..26cd6ae 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -68,9 +68,6 @@ `(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name)) (a (@ (href "https://cadence.moe/contact")) "Cadence") - ". Proudly hosted by " - (a (@ (href "http://alphamethyl.barr0w.net")) - "Barrow Network Solutions" (sup "XD")) ".") `(p ,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name))))) @@ -88,11 +85,13 @@ ;; generate a notice with a link if a fandom wiki has a replacement as part of NIWA or similar ;; if the wiki has no replacement, display nothing -(define (extwiki-notice wikiname title) +(define (extwiki-notice wikiname title req user-cookies) (define xt (findf (λ (item) (member wikiname (extwiki^-wikinames item))) extwikis)) (cond/var [xt - (let* ([group (hash-ref extwiki-groups (extwiki^-group xt))] + (let* ([seen? (member wikiname (user-cookies^-notices user-cookies))] + [aside-class (if seen? "niwa__notice niwa--seen" "niwa__notice")] + [group (hash-ref extwiki-groups (extwiki^-group xt))] [search-page (format "/Special:Search?~a" (params->query `(("search" . ,title) ("go" . "Go"))))] @@ -103,7 +102,7 @@ [props (extwiki-props^ go)]) (cond [(eq? (extwiki^-banner xt) 'default) - `(aside (@ (class "niwa__notice")) + `(aside (@ (class ,aside-class)) (h1 (@ (class "niwa__header")) ,(extwiki^-name xt) " has its own website separate from Fandom.") (a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,(extwiki^-name xt) " →") (div (@ (class "niwa__cols")) @@ -117,7 +116,10 @@ `(,@(for/list ([link (extwiki-group^-links group)]) `(a (@ (href ,(cdr link))) ,(car link))) "This notice is from BreezeWiki" - (a (@ (href "https://docs.breezewiki.com/Reporting_Bugs.html")) "Feedback?")) + (a (@ (rel "nofollow") + (class "niwa__got-it") + (href ,(user-cookies-setter-url/add-notice req user-cookies wikiname))) + "OK, got it")) " / "))) (div (@ (class "niwa__right")) (img (@ (class "niwa__logo") (src ,(extwiki^-logo xt)))))))] @@ -225,7 +227,7 @@ (div (@ (class "fandom-community-header__background tileHorizontally header"))) (div (@ (class "page")) (main (@ (class "page__main")) - ,(extwiki-notice wikiname title) + ,(extwiki-notice wikiname title req user-cookies) (div (@ (class "custom-top")) (h1 (@ (class "page-title")) ,title) (nav (@ (class "sitesearch")) diff --git a/src/data.rkt b/src/data.rkt index 5aba2c2..63c7f03 100644 --- a/src/data.rkt +++ b/src/data.rkt @@ -1,6 +1,7 @@ #lang racket/base (require racket/list racket/match + racket/string web-server/http/request-structs net/url-string (only-in net/cookies/server cookie-header->alist cookie->set-cookie-header make-cookie) @@ -27,7 +28,8 @@ user-cookies-getter user-cookies-default user-cookies-setter - user-cookies-setter-url) + user-cookies-setter-url + user-cookies-setter-url/add-notice) (struct siteinfo^ (sitename basepage license) #:transparent) (struct license^ (text url) #:transparent) @@ -90,8 +92,8 @@ ;; then no matter what, return the best information we have so far this-data)) -(struct user-cookies^ (theme) #:prefab) -(define user-cookies-default (user-cookies^ 'default)) +(struct user-cookies^ (theme notices) #:prefab) +(define user-cookies-default (user-cookies^ 'default '())) (define (user-cookies-getter req) (define cookie-header (headers-assq* #"cookie" (request-headers/raw req))) (define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null)) @@ -100,16 +102,29 @@ (match pair [(cons "theme" (and theme (or "light" "dark" "default"))) (values 'theme (string->symbol theme))] + [(cons "notices" notices) + (values 'notices (string-split notices "|"))] [_ (values #f #f)]))) (user-cookies^ - (hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default)))) + (hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default)) + (hash-ref cookies-hash 'notices (user-cookies^-notices user-cookies-default)))) (define (user-cookies-setter user-cookies) (map (λ (c) (header #"Set-Cookie" (cookie->set-cookie-header c))) (list (make-cookie "theme" (symbol->string (user-cookies^-theme user-cookies)) + #:path "/" + #:max-age (* 60 60 24 365 10)) + (make-cookie "notices" (string-join (user-cookies^-notices user-cookies) "|") + #:path "/" #:max-age (* 60 60 24 365 10))))) (define (user-cookies-setter-url req new-settings) (format "/set-user-settings?~a" (params->query `(("next_location" . ,(url->string (request-uri req))) - ("new_settings" . ,(format "~a" new-settings)))))) + ("new_settings" . ,(format "~s" new-settings)))))) + +(define (user-cookies-setter-url/add-notice req user-cookies notice-name) + (user-cookies-setter-url + req + (struct-copy user-cookies^ user-cookies + [notices (cons notice-name (user-cookies^-notices user-cookies))]))) diff --git a/static/main.css b/static/main.css index bcd612b..a8e9703 100644 --- a/static/main.css +++ b/static/main.css @@ -431,6 +431,23 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ font-size: 14px; text-align: right; } +/* more compact notice after it's been seen the first time */ +.niwa--seen { + padding: 1.5vw 2vw 2vw; + overflow-y: auto; + max-height: min(280px, 33vh); + font-size: 17px; +} +.niwa--seen .niwa__header { + font-size: 26px; +} +.niwa--seen .niwa__go { + padding: 10px 18px; + font-size: 20px; +} +.niwa--seen .niwa__got-it { + display: none; +} /* media queries */ From 14930f18dc73457f1cb6ef677285333a112a048e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Jul 2024 02:32:39 +1200 Subject: [PATCH 20/27] Save even more vertical space --- static/main.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/static/main.css b/static/main.css index a8e9703..5b3e7e2 100644 --- a/static/main.css +++ b/static/main.css @@ -437,6 +437,8 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */ overflow-y: auto; max-height: min(280px, 33vh); font-size: 17px; + margin-top: -2vw; + margin-bottom: 12px; } .niwa--seen .niwa__header { font-size: 26px; From 8f0caa913240df0e797c587d460ab460f75528f3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 1 Jul 2024 02:45:51 +1200 Subject: [PATCH 21/27] Add Enter the Gungeon wiki.gg redirect --- src/extwiki-data.rkt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 29997c4..f190e5f 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -512,6 +512,7 @@ (extwiki^ '("doom") 'empty 'empty "DoomWiki.org" "https://doomwiki.org/wiki/Entryway" #f #f) (extwiki^ '("dreamscaper") 'empty 'empty "Official Dreamscaper Wiki" "https://dreamscaper.wiki.gg/wiki/Dreamscaper_Wiki" #f #f) (extwiki^ '("elderscrolls") 'empty 'empty "UESP" "https://en.uesp.net/wiki/Main_Page" #f #f) + (extwiki^ '("enterthegungeon" "exit-the-gungeon" "enter-the-gungeon-archive") 'empty 'empty "Official Enter The Gungeon Wiki" "https://enterthegungeon.wiki.gg/wiki/Enter_the_Gungeon_Wiki" "https://enterthegungeon.wiki.gg/images/e/e6/Site-logo.png" #f) (extwiki^ '("fiend-folio") 'empty 'empty "Official Fiend Folio Wiki" "https://fiendfolio.wiki.gg/wiki/Fiend_Folio_Wiki" #f #f) (extwiki^ '("foxhole") 'empty 'empty "Foxhole Wiki" "https://foxhole.wiki.gg/wiki/Foxhole_Wiki" #f #f) (extwiki^ '("have-a-nice-death") 'empty 'empty "Have a Nice Death Wiki" "https://haveanicedeath.wiki.gg/wiki/Have_a_Nice_Death_Wiki" #f #f) From 49682b23207443a7766853f4afd90e0dbd4520c8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 3 Jul 2024 23:37:44 +1200 Subject: [PATCH 22/27] Fix for Racket 8.13 --- src/config.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rkt b/src/config.rkt index fec546b..b1afe0a 100644 --- a/src/config.rkt +++ b/src/config.rkt @@ -104,7 +104,7 @@ ; all values here are optimised for maximum prettiness (parameterize ([pretty-print-columns 80]) (display "config: ") - (pretty-write ((inst sort (Pairof Symbol String)) + (pretty-write ((inst sort (Pairof Symbol String) Symbol) (hash->list (make-immutable-hasheq combined-alist)) symbol Date: Thu, 4 Jul 2024 19:21:17 +1200 Subject: [PATCH 23/27] Add Granblue redirect --- src/extwiki-data.rkt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index f190e5f..8971640 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -89,7 +89,7 @@ (extwiki-group^ "Misc" '(("This wiki doesn't have a description yet. Add one?" . "https://docs.breezewiki.com/Reporting_Bugs.html")) - #f))) + (λ (props) '())))) ;; wikiname, niwa-name, url, logo-url (struct extwiki^ (wikinames banner group name home logo description) #:transparent) @@ -497,6 +497,15 @@ (λ (props) `())) + (extwiki^ + '("granblue") 'empty + 'empty + "Granblue Fantasy Wiki" + "https://gbf.wiki/" + "https://gbf.wiki/images/1/18/Vyrnball.png?0704c" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 1e3451a990e64fc75b76ee51bd626ccd57e28127 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 14 Jul 2024 23:25:52 +1200 Subject: [PATCH 24/27] Add HELLMET wiki --- src/extwiki-data.rkt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 8971640..4a2cca2 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -506,6 +506,15 @@ (λ (props) `())) + (extwiki^ + '("hellmet-roblox") 'empty + 'empty + "HELLMET Wiki" + "https://hellmet.miraheze.org/wiki/Main_Page" + "https://static.miraheze.org/hellmetwiki/thumb/c/ce/Hellmet_Wiki_Logo.png/135px-Hellmet_Wiki_Logo.png" + (λ (props) + `())) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 8db91d5e32c1a8671d4d7537e381611e5732e8f9 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 14 Jul 2024 23:38:30 +1200 Subject: [PATCH 25/27] Add Rainverse wiki redirect --- src/extwiki-data.rkt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index 4a2cca2..e283dfb 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -85,6 +85,13 @@ ("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/")) (λ (props) '())) + 'Rainverse + (extwiki-group^ + "Rainverse" + '(("Forking announcement" . "https://transfem.social/notes/9qsqdkmqi78e01bh")) + (λ (props) + '())) + 'empty (extwiki-group^ "Misc" @@ -515,6 +522,17 @@ (λ (props) `())) + (extwiki^ + '("rain-web-comic") 'default + 'empty + "Rainverse Wiki" + "https://rainverse.wiki/wiki/Main_Page" + "https://static.miraheze.org/rainversewiki/2/2c/Rain_comic_cover.png" + (λ (props) + `((p "We have a newly-migrated Rainverse Wiki which escaped from Fandom! Rain is the comic that helped me figure out my gender, so I am really glad to have a wiki on a non-evil host.") + (p "Please stop using the abandoned copy of Rain Wiki on Fandom. Fandom is still \"training\" a generator which adds procedurally-generated bullshit to articles, with no way for users to remove or correct it, and they're demanding volunteer wiki admins waste time \"vetting\" the procedurally-generated BS for accuracy. As Jocelyn herself said, \"fuck Fandom forever.\"") + (p "If you are interested, please add more articles related to other Rainverse stories.")))) + ;; fandom wikinames * empty * empty * Name * Home Page (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) From 97c4e54f38558f67abb1a1cc5baec23fa9cc0b21 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 10 Aug 2024 15:04:13 +1200 Subject: [PATCH 26/27] Fix Tardis Wiki metadata --- src/extwiki-data.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extwiki-data.rkt b/src/extwiki-data.rkt index e283dfb..a8ee159 100644 --- a/src/extwiki-data.rkt +++ b/src/extwiki-data.rkt @@ -443,9 +443,9 @@ (extwiki^ '("tardis") 'default 'Tardis - "Tardis Data Core" + "TARDIS Wiki" "https://tardis.wiki/wiki/Doctor_Who_Wiki" - "https://tardis.wiki/images/Tardis_images/e/e6/Site-logo.png" + "https://tardis.wiki/w/images/Tardis_Images/e/e6/Site-logo.png" (λ (props) `())) From 443f1eecbc4c15c8038920027e62fccbdcb0bbe7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 23 Oct 2024 22:52:00 +1300 Subject: [PATCH 27/27] Add user agent and detect blocked pages --- src/fandom-request.rkt | 40 +++++++++++++++++++++++++++++++++------- src/page-wiki.rkt | 28 ++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/fandom-request.rkt b/src/fandom-request.rkt index 966eeee..c306b04 100644 --- a/src/fandom-request.rkt +++ b/src/fandom-request.rkt @@ -1,12 +1,16 @@ #lang typed/racket/base -(require "config.rkt" +(require racket/format + racket/string + "config.rkt" "../lib/url-utils.rkt") (define-type Headers (HashTable Symbol (U Bytes String))) (require/typed net/http-easy [#:opaque Timeout-Config timeout-config?] [#:opaque Response response?] [#:opaque Session session?] + [response-status-code (Response -> Natural)] [current-session (Parameter Session)] + [current-user-agent (Parameter (U Bytes String))] [make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)] [get ((U Bytes String) [#:close? Boolean] @@ -22,19 +26,41 @@ fandom-get-api timeouts) +(unless (string-contains? (~a (current-user-agent)) "BreezeWiki") + (current-user-agent + (format "BreezeWiki/1.0 (~a) ~a" + (if (config-true? 'canonical_origin) + (config-get 'canonical_origin) + "local") + (current-user-agent)))) + (define timeouts (make-timeout-config #:lease 5 #:connect 5)) +(: last-failure Flonum) +(define last-failure 0.0) +(: stored-failure (Option Response)) +(define stored-failure #f) +(define failure-persist-time 30000) + (: no-headers Headers) (define no-headers '#hasheq()) (: fandom-get (String String [#:headers (Option Headers)] -> Response)) (define (fandom-get wikiname path #:headers [headers #f]) - (define dest-url (string-append "https://www.fandom.com" path)) - (define host (string-append wikiname ".fandom.com")) - (log-outgoing wikiname path) - (get dest-url - #:timeouts timeouts - #:headers (hash-set (or headers no-headers) 'Host host))) + (or + (and ((current-inexact-milliseconds) . < . (+ last-failure failure-persist-time)) stored-failure) + (let () + (define dest-url (string-append "https://www.fandom.com" path)) + (define host (string-append wikiname ".fandom.com")) + (log-outgoing wikiname path) + (define res + (get dest-url + #:timeouts timeouts + #:headers (hash-set (or headers no-headers) 'Host host))) + (when (memq (response-status-code res) '(403 406)) + (set! last-failure (current-inexact-milliseconds)) + (set! stored-failure res)) + res))) (: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response)) (define (fandom-get-api wikiname params #:headers [headers #f]) diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index f16792c..da63617 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -18,6 +18,7 @@ "config.rkt" "data.rkt" "fandom-request.rkt" + "../lib/archive-file-mappings.rkt" "../lib/pure-utils.rkt" "../lib/syntax.rkt" "../lib/thread-utils.rkt" @@ -37,8 +38,9 @@ (define (page-wiki req) (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define segments (map path/param-path (cdr (url-path (request-uri req))))) (define user-cookies (user-cookies-getter req)) - (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) + (define path (string-join (cdr segments) "/")) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define-values (dest-res siteinfo) @@ -101,9 +103,31 @@ (write-html body out))))))] [(eq? 404 (easy:response-status-code dest-res)) (next-dispatcher)] + [(memq (easy:response-status-code dest-res) '(403 406)) + (response-handler + (define body + (generate-wiki-page + `(div + (p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.") + (p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.") + (p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom.")))) + #:req req + #:source-url source-url + #:wikiname wikiname + #:title (url-segments->guess-title segments) + #:siteinfo siteinfo)) + (response/output + #:code 503 + #:headers (build-headers + always-headers + (header #"Retry-After" #"30") + (header #"Cache-Control" #"max-age=30, public") + (header #"Refresh" #"35")) + (λ (out) + (write-html body out))))] [else (response-handler - (error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" + (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a" wikiname path (easy:response-status-code dest-res)