From 9611b8c1644c90d0c4bb916cecd28e787752a157 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 15 Nov 2023 23:46:40 +1300 Subject: [PATCH] Disable category, file, search pages in offline mode --- src/page-category.rkt | 116 ++++++++++++++++++++++-------------------- src/page-file.rkt | 92 +++++++++++++++++---------------- src/page-search.rkt | 86 ++++++++++++++++--------------- 3 files changed, 156 insertions(+), 138 deletions(-) diff --git a/src/page-category.rkt b/src/page-category.rkt index 213d423..b595b09 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -65,64 +65,70 @@ (define (page-category req) (response-handler - (define wikiname (path/param-path (first (url-path (request-uri req))))) - (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-category)) + (cond + [(config-true? 'feature_offline::only) + (response/output #:code 503 + #:headers (build-headers always-headers) + (λ (out) (write-html '(p "Sorry, category pages are temporarily disabled. I hope to have them back soon.") out)))] + [else + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-category)) - (define-values (members-data page-data siteinfo) - (thread-values - (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "categorymembers") - ("cmtitle" . ,prefixed-category) - ("cmlimit" . "max") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) - (λ () - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "parse") - ("page" . ,prefixed-category) - ("prop" . "text|headhtml|langlinks") - ("formatversion" . "2") - ("format" . "json"))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) - (λ () - (siteinfo-fetch wikiname)))) + (define-values (members-data page-data siteinfo) + (thread-values + (λ () + (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "query") + ("list" . "categorymembers") + ("cmtitle" . ,prefixed-category) + ("cmlimit" . "max") + ("formatversion" . "2") + ("format" . "json"))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)) + (λ () + (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "parse") + ("page" . ,prefixed-category) + ("prop" . "text|headhtml|langlinks") + ("formatversion" . "2") + ("format" . "json"))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)) + (λ () + (siteinfo-fetch wikiname)))) - (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) - (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) - (define page (html->xexp page-html)) - (define head-data ((head-data-getter wikiname) page-data)) - (define body (generate-results-page - #:req req - #:source-url source-url - #:wikiname wikiname - #:title title - #:members-data members-data - #:page page - #:head-data head-data - #:siteinfo siteinfo)) + (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category))) + (define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) + (define page (html->xexp page-html)) + (define head-data ((head-data-getter wikiname) page-data)) + (define body (generate-results-page + #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:members-data members-data + #:page page + #:head-data head-data + #:siteinfo siteinfo)) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - #:headers (build-headers always-headers) - (λ (out) - (write-html body out))))) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + #:headers (build-headers always-headers) + (λ (out) + (write-html body out)))]))) (module+ test (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") (generate-results-page diff --git a/src/page-file.rkt b/src/page-file.rkt index 2a7332c..11f5407 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -104,50 +104,56 @@ (define (page-file req) (response-handler - (define wikiname (path/param-path (first (url-path (request-uri req))))) - (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-title)) + (cond + [(config-true? 'feature_offline::only) + (response/output #:code 503 + #:headers (build-headers always-headers) + (λ (out) (write-html '(p "Sorry, file pages are temporarily disabled. I hope to have them back soon.") out)))] + [else + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-title)) - (define-values (media-detail siteinfo) - (thread-values - (λ () - (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . ,prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)) - (λ () - (siteinfo-fetch wikiname)))) - (if (not (jp "/exists" media-detail #f)) - (next-dispatcher) - (response-handler - (define file-title (jp "/fileTitle" media-detail "")) - (define title - (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) - (define image-content-type - (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) - #f - (url-content-type (jp "/imageUrl" media-detail)))) - (define body - (generate-results-page #:req req - #:source-url source-url - #:wikiname wikiname - #:title title - #:media-detail media-detail - #:image-content-type image-content-type - #:siteinfo siteinfo)) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output #:code 200 - #:headers (build-headers always-headers) - (λ (out) (write-html body out))))))) + (define-values (media-detail siteinfo) + (thread-values + (λ () + (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)) + (λ () + (siteinfo-fetch wikiname)))) + (if (not (jp "/exists" media-detail #f)) + (next-dispatcher) + (response-handler + (define file-title (jp "/fileTitle" media-detail "")) + (define title + (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) + (define image-content-type + (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) + #f + (url-content-type (jp "/imageUrl" media-detail)))) + (define body + (generate-results-page #:req req + #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:siteinfo siteinfo)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output #:code 200 + #:headers (build-headers always-headers) + (λ (out) (write-html body out)))))]))) (module+ test (parameterize ([(config-parameter 'strict_proxy) "true"]) (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") diff --git a/src/page-search.rkt b/src/page-search.rkt index e4960d8..665c104 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -68,48 +68,54 @@ (define (page-search req) ;; this just means, catch any errors and display them in the browser. it's a function somewhere else (response-handler - ;; the URL will look like "/minecraft/wiki/Special:Search?q=Spawner" - ;; grab the first part to use as the wikiname, in this case, "minecraft" - (define wikiname (path/param-path (first (url-path (request-uri req))))) - ;; grab the part after ?q= which is the search terms - (define query (dict-ref (url-query (request-uri req)) 'q #f)) - ;; constructing the URL where I want to get fandom data from... - (define origin (format "https://~a.fandom.com" wikiname)) - ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json - (define dest-url - (format "~a/api.php?~a" - origin - (params->query `(("action" . "query") - ("list" . "search") - ("srsearch" . ,query) - ("formatversion" . "2") - ("format" . "json"))))) + (cond + [(config-true? 'feature_offline::only) + (response/output #:code 503 + #:headers (build-headers always-headers) + (λ (out) (write-html '(body (p "Sorry, full search is temporarily broken, but I have a plan to fix it.") (p "In the meantime, please use the popup search suggestions below the search box.")) out)))] + [else + ;; the URL will look like "/minecraft/wiki/Special:Search?q=Spawner" + ;; grab the first part to use as the wikiname, in this case, "minecraft" + (define wikiname (path/param-path (first (url-path (request-uri req))))) + ;; grab the part after ?q= which is the search terms + (define query (dict-ref (url-query (request-uri req)) 'q #f)) + ;; constructing the URL where I want to get fandom data from... + (define origin (format "https://~a.fandom.com" wikiname)) + ;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json + (define dest-url + (format "~a/api.php?~a" + origin + (params->query `(("action" . "query") + ("list" . "search") + ("srsearch" . ,query) + ("formatversion" . "2") + ("format" . "json"))))) - ;; simultaneously get the search results from the fandom API, as well as information about the wiki as a whole (its license, icon, name) - (define-values (dest-res siteinfo) - (thread-values - (λ () - (log-outgoing dest-url) - (easy:get dest-url #:timeouts timeouts)) ;; HTTP request to dest-url for search results - (λ () - (siteinfo-fetch wikiname)))) ;; helper function in another file to get information about the wiki + ;; simultaneously get the search results from the fandom API, as well as information about the wiki as a whole (its license, icon, name) + (define-values (dest-res siteinfo) + (thread-values + (λ () + (log-outgoing dest-url) + (easy:get dest-url #:timeouts timeouts)) ;; HTTP request to dest-url for search results + (λ () + (siteinfo-fetch wikiname)))) ;; helper function in another file to get information about the wiki - ;; search results are a JSON string. parse JSON into racket data structures - (define data (easy:response-json dest-res)) - ;; calling my generate-results-page function with the information so far in order to get a big fat x-expression - ;; big fat x-expression goes into the body variable - (define body (generate-results-page req dest-url wikiname query data #:siteinfo siteinfo)) - ;; error checking - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - ;; convert body to HTML and send to browser - (response/output - #:code 200 - #:headers (build-headers always-headers) - (λ (out) - (write-html body out))))) + ;; search results are a JSON string. parse JSON into racket data structures + (define data (easy:response-json dest-res)) + ;; calling my generate-results-page function with the information so far in order to get a big fat x-expression + ;; big fat x-expression goes into the body variable + (define body (generate-results-page req dest-url wikiname query data #:siteinfo siteinfo)) + ;; error checking + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + ;; convert body to HTML and send to browser + (response/output + #:code 200 + #:headers (build-headers always-headers) + (λ (out) + (write-html body out)))]))) (module+ test (parameterize ([(config-parameter 'feature_offline::only) "false"]) (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")