From 5fb7301ce1bc29c4bdb4e2be342daa19f29674d8 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:35:35 +0700 Subject: [PATCH 01/46] Add support for File: pages Fixes https://lists.sr.ht/~cadence/breezewiki-discuss/%3Cb2835a70-5118-4df0-90c9-4333486a4b69%40nixnetmail.com%3E --- src/dispatcher-tree.rkt | 1 + src/page-file.rkt | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 src/page-file.rkt diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index f2c1412..b68cf9c 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -45,6 +45,7 @@ (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) diff --git a/src/page-file.rkt b/src/page-file.rkt new file mode 100644 index 0000000..24dd5ae --- /dev/null +++ b/src/page-file.rkt @@ -0,0 +1,117 @@ +#lang racket/base +(require racket/dict + racket/list + racket/match + racket/string + (prefix-in easy: net/http-easy) + ; html libs + html-parsing + html-writing + ; web server libs + net/url + web-server/http + (only-in web-server/dispatchers/dispatch next-dispatcher) + #;(only-in web-server/http/redirect redirect-to) + "application-globals.rkt" + "config.rkt" + "data.rkt" + "page-wiki.rkt" + "syntax.rkt" + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide + page-file) + +;(module+ test +; (require rackunit) +; (define category-json-data +; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) + +(define (generate-results-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:license [license #f]) + (define video-embed-code (jp "/videoEmbedCode" media-detail "")) + (define raw-image-url (jp "/rawImageUrl" media-detail)) + (define image-url (jp "/imageUrl" media-detail raw-image-url)) + (define username (jp "/userName" media-detail)) + (define user-page-url (jp "/userPageUrl" media-detail)) + (define is-posted-in (jp "/isPostedIn" media-detail #f)) + (define smaller-article-list (jp "/smallerArticleList" media-detail)) + (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) + (define image-description (jp "/imageDescription" media-detail #f)) + (generate-wiki-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:license license + `(div + (p + ,(if (non-empty-string? video-embed-code) + `(span (a (@ (href (u-proxy-url raw-image-url))) "View original file") ". ") + `(return-no-element)) + "Added by " + (a (@ (href (u-proxy-url user-page-url))) username) + "." + ,(if is-posted-in + `(span " Posted in " + ,@(map + (λ (article) + (define page-path (jp "/title" result)) + (define title (jp "/titleText" result page-path)) + `(a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title)) + smaller-article-list)) + ,(if (= article-list-is-smaller 1) + "…" + ".")))) +; ,(update-tree-wiki image-description wikiname) + )) + +(define (page-file req) + (response-handler + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-title)) + + (thread-let + ([media-detail (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") + ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [license (license-auto wikiname)]) + + (define title (jp "/fileTitle" page-data media-detail))) + (define body (generate-results-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:license license)) + + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + (λ (out) + (write-html body out)))))) +;(module+ test +; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") +; (generate-results-page +; #:source-url "" +; #:wikiname "test" +; #:title "Category:Items" +; #:members-data category-json-data +; #:page '(div "page text")))))) From 0148c1c3f922e7a76e4e4d237405cb3d0f956700 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:42:00 +0700 Subject: [PATCH 02/46] idfk --- src/page-file.rkt | 66 ++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 24dd5ae..608e9c8 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -72,41 +72,43 @@ )) (define (page-file req) - (response-handler - (define wikiname (path/param-path (first (url-path (request-uri req))))) - (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) - (define origin (format "https://~a.fandom.com" wikiname)) - (define source-url (format "~a/wiki/~a" origin prefixed-title)) + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-title)) - (thread-let - ([media-detail (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") - ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)] - [license (license-auto wikiname)]) + (thread-let + ([media-detail (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") + ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [license (license-auto wikiname)]) - (define title (jp "/fileTitle" page-data media-detail))) - (define body (generate-results-page - #:source-url source-url - #:wikiname wikiname - #:title title - #:media-detail media-detail - #:license license)) + (if (not (jp "/exists" media-detail #f)) + (next-dispatcher) + (response-handler + (define title (jp "/fileTitle" media-detail))) + (define body (generate-results-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:license license)) - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - (λ (out) - (write-html body out)))))) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output + #:code 200 + (λ (out) + (write-html body out)))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") ; (generate-results-page From 00da2efd7a37a7a056297fdaf61dd7d57e7eb6b2 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:42:41 +0700 Subject: [PATCH 03/46] article... --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 608e9c8..ab27146 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -60,8 +60,8 @@ `(span " Posted in " ,@(map (λ (article) - (define page-path (jp "/title" result)) - (define title (jp "/titleText" result page-path)) + (define page-path (jp "/title" article)) + (define title (jp "/titleText" article page-path)) `(a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) ,title)) smaller-article-list)) From 600338aef07af419876c60203f16dd08d99be801 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:44:56 +0700 Subject: [PATCH 04/46] ` --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index ab27146..a229274 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -65,7 +65,7 @@ `(a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) ,title)) smaller-article-list)) - ,(if (= article-list-is-smaller 1) + `(if (= article-list-is-smaller 1) "…" ".")))) ; ,(update-tree-wiki image-description wikiname) From fe87ff26c5bdef7e5f678cbed2592bb4c652feb7 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:57:33 +0700 Subject: [PATCH 05/46] b --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index a229274..d57c839 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -88,7 +88,7 @@ (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] - [license (license-auto wikiname)]) + [license (license-auto wikiname)])) (if (not (jp "/exists" media-detail #f)) (next-dispatcher) @@ -108,7 +108,7 @@ (response/output #:code 200 (λ (out) - (write-html body out)))))) + (write-html body out))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") ; (generate-results-page From 8b8a9c5e099d8bf37c44b227a4f8961faa23d93c Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 16:12:30 +0700 Subject: [PATCH 06/46] bb --- src/page-file.rkt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d57c839..84ff02f 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -78,22 +78,22 @@ (define source-url (format "~a/wiki/~a" origin prefixed-title)) (thread-let - ([media-detail (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") - ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)] - [license (license-auto wikiname)])) + ([media-detail (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") + ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [license (license-auto wikiname)])) (if (not (jp "/exists" media-detail #f)) (next-dispatcher) (response-handler - (define title (jp "/fileTitle" media-detail))) + (define title (jp "/fileTitle" media-detail)) (define body (generate-results-page #:source-url source-url #:wikiname wikiname @@ -108,7 +108,7 @@ (response/output #:code 200 (λ (out) - (write-html body out))))) + (write-html body out)))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") ; (generate-results-page From 01363b366af1c70a2e89b7811fbe41bb690e40fd Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 16:15:03 +0700 Subject: [PATCH 07/46] bbb --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 84ff02f..f77d54f 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -88,7 +88,7 @@ (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] - [license (license-auto wikiname)])) + [license (license-auto wikiname)]) (if (not (jp "/exists" media-detail #f)) (next-dispatcher) @@ -108,7 +108,7 @@ (response/output #:code 200 (λ (out) - (write-html body out)))))) + (write-html body out))))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") ; (generate-results-page From b7424a76499eea7dc406f5aebf321fc7d30d694b Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 16:16:13 +0700 Subject: [PATCH 08/46] f --- breezewiki.rkt | 1 + dist.rkt | 1 + 2 files changed, 2 insertions(+) diff --git a/breezewiki.rkt b/breezewiki.rkt index dfb405e..58ec23c 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -38,6 +38,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher)))) diff --git a/dist.rkt b/dist.rkt index a626695..c0dbd1b 100644 --- a/dist.rkt +++ b/dist.rkt @@ -27,6 +27,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher))) From ce496dbba1a6667824cbc7992b5cec3cdf5bb1d8 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 16:17:10 +0700 Subject: [PATCH 09/46] ff --- breezewiki.rkt | 1 + dist.rkt | 1 + 2 files changed, 2 insertions(+) diff --git a/breezewiki.rkt b/breezewiki.rkt index 58ec23c..3fc9b8f 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -19,6 +19,7 @@ (require-reloadable "src/page-static.rkt" static-dispatcher) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) +(require-reloadable "src/page-file.rkt" page-file) (reload!) diff --git a/dist.rkt b/dist.rkt index c0dbd1b..805df48 100644 --- a/dist.rkt +++ b/dist.rkt @@ -13,6 +13,7 @@ (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) (require (only-in "src/page-wiki.rkt" page-wiki)) +(require (only-in "src/page-file.rkt" page-file)) (serve/launch/wait #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f) From f6e011e687c6697d2c10f1d673be1176d74bb58a Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 16:21:02 +0700 Subject: [PATCH 10/46] , --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index f77d54f..b9b742b 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -84,7 +84,7 @@ (params->query `(("format" . "json") ("controller" . "Lightbox") ("method" . "getMediaDetail") - ("fileTitle" . prefixed-title))))) + ("fileTitle" . ,prefixed-title))))) (log-outgoing dest-url) (define dest-res (easy:get dest-url #:timeouts timeouts)) (easy:response-json dest-res)] From 80459eccad3265a3d4203b80abd8e634096185d8 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 17:09:13 +0700 Subject: [PATCH 11/46] H --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index b9b742b..466de49 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -51,10 +51,10 @@ `(div (p ,(if (non-empty-string? video-embed-code) - `(span (a (@ (href (u-proxy-url raw-image-url))) "View original file") ". ") + `(span (a (@ (href ,(u-proxy-url raw-image-url))) "View original file") ". ") `(return-no-element)) "Added by " - (a (@ (href (u-proxy-url user-page-url))) username) + (a (@ (href ,(u-proxy-url user-page-url))) ,username) "." ,(if is-posted-in `(span " Posted in " From 7d8763b308ded8e5af55f316cba7be0323c7c529 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 17:34:11 +0700 Subject: [PATCH 12/46] ? --- src/page-file.rkt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 466de49..d3b808d 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -51,8 +51,8 @@ `(div (p ,(if (non-empty-string? video-embed-code) - `(span (a (@ (href ,(u-proxy-url raw-image-url))) "View original file") ". ") - `(return-no-element)) + `"" + `(span (a (@ (href ,(u-proxy-url raw-image-url))) "View original file") ". ")) "Added by " (a (@ (href ,(u-proxy-url user-page-url))) ,username) "." @@ -62,10 +62,14 @@ (λ (article) (define page-path (jp "/title" article)) (define title (jp "/titleText" article page-path)) - `(a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title)) + `(span + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title) + `(if (eq? (car smaller-article-list) article)) + "" + ", ")) smaller-article-list)) - `(if (= article-list-is-smaller 1) + `(if (eq? article-list-is-smaller 1) "…" ".")))) ; ,(update-tree-wiki image-description wikiname) From 940b6e01db12a73b97aef087b5132b21821c00b8 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 17:56:56 +0700 Subject: [PATCH 13/46] t --- src/page-file.rkt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d3b808d..d112e10 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -38,7 +38,6 @@ (define raw-image-url (jp "/rawImageUrl" media-detail)) (define image-url (jp "/imageUrl" media-detail raw-image-url)) (define username (jp "/userName" media-detail)) - (define user-page-url (jp "/userPageUrl" media-detail)) (define is-posted-in (jp "/isPostedIn" media-detail #f)) (define smaller-article-list (jp "/smallerArticleList" media-detail)) (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) @@ -54,7 +53,7 @@ `"" `(span (a (@ (href ,(u-proxy-url raw-image-url))) "View original file") ". ")) "Added by " - (a (@ (href ,(u-proxy-url user-page-url))) ,username) + (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) "." ,(if is-posted-in `(span " Posted in " @@ -63,15 +62,16 @@ (define page-path (jp "/title" article)) (define title (jp "/titleText" article page-path)) `(span - (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title) - `(if (eq? (car smaller-article-list) article)) + ,(if (eq? (car smaller-article-list) article) "" - ", ")) - smaller-article-list)) - `(if (eq? article-list-is-smaller 1) + ", ") + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title))) + smaller-article-list) + ,(if (eq? article-list-is-smaller 1) "…" - ".")))) + ".")) + `""))) ; ,(update-tree-wiki image-description wikiname) )) From db0158f237834209523fb96924b70080d1ed4e82 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:11:38 +0700 Subject: [PATCH 14/46] image desc --- src/page-file.rkt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d112e10..d6b5f33 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -72,8 +72,9 @@ "…" ".")) `""))) -; ,(update-tree-wiki image-description wikiname) - )) + ,(if (string? image-description) + `(update-tree-wiki (html->xexp image-description) wikiname) + `"")) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From d336307921d3f75e3ad336b38ef048aeaab3cb97 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:13:19 +0700 Subject: [PATCH 15/46] ) --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d6b5f33..e88ffbe 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -71,7 +71,7 @@ ,(if (eq? article-list-is-smaller 1) "…" ".")) - `""))) + `"")) ,(if (string? image-description) `(update-tree-wiki (html->xexp image-description) wikiname) `"")) From 21f4f7acbc76d8dadc777157f8cc9dfbbf2b4a57 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:14:11 +0700 Subject: [PATCH 16/46] )) --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index e88ffbe..389f561 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -74,7 +74,7 @@ `"")) ,(if (string? image-description) `(update-tree-wiki (html->xexp image-description) wikiname) - `"")) + `""))) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From c2c699ec8f8257f9e69e9ff3f29a09c6ac15a151 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:15:02 +0700 Subject: [PATCH 17/46] ))) --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 389f561..8a166cd 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -74,7 +74,7 @@ `"")) ,(if (string? image-description) `(update-tree-wiki (html->xexp image-description) wikiname) - `""))) + `"")))) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From 1558f9f541c823276ba7bdc89c28f20111ffbef1 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:16:38 +0700 Subject: [PATCH 18/46] preprocess --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 8a166cd..f501eb9 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -73,7 +73,7 @@ ".")) `"")) ,(if (string? image-description) - `(update-tree-wiki (html->xexp image-description) wikiname) + `(update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) `"")))) (define (page-file req) From fd4d65abb24f18115455cffe0bd3370bc3cdd80c Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 21:37:19 +0700 Subject: [PATCH 19/46] ` --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index f501eb9..962144e 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -73,8 +73,8 @@ ".")) `"")) ,(if (string? image-description) - `(update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) - `"")))) + (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) + "")))) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) From 04ed98fb4b1d019f7eb5ec3330a5e1b8857d2a74 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:26:17 +0700 Subject: [PATCH 20/46] lost --- src/page-file.rkt | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 962144e..dfd3024 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -28,6 +28,30 @@ ; (define category-json-data ; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) +(define (html-from-url image-url) + ([media-detail (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") + ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + (log-outgoing image-url) + (define dest-res (easy:head image-url #:timeouts timeouts)) + (define content-type (response-headers-ref dest-res "Content-Type")) + (cond + [(eq? content-type #f) ""] + [(regexp-match? #rx"^image/" content-type) + (img (@ (src ,(u-proxy-url image-url))))] + [(regexp-match? #rx"^audio/" content-type) + (audio (@ (src ,(u-proxy-url image-url)) (controls)))] + [(regexp-match? #rx"^video/" content-type) + (video (@ (src ,(u-proxy-url image-url)) (controls)))] + [else ""])) + (define (generate-results-page #:source-url source-url #:wikiname wikiname @@ -42,16 +66,22 @@ (define smaller-article-list (jp "/smallerArticleList" media-detail)) (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) (define image-description (jp "/imageDescription" media-detail #f)) + (define maybe-proxied-raw-image-url (if (config-true? 'strict_proxy) + (u-proxy-url raw-image-url) + raw-image-url)) (generate-wiki-page #:source-url source-url #:wikiname wikiname #:title title #:license license `(div + ,(if (non-empty-string? video-embed-code) + (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) + `(html-from-url image-url)) (p ,(if (non-empty-string? video-embed-code) `"" - `(span (a (@ (href ,(u-proxy-url raw-image-url))) "View original file") ". ")) + `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) "Added by " (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) "." From ff18bda668c6f8d37121dedf2f545f205de8b24f Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:27:28 +0700 Subject: [PATCH 21/46] oops --- src/page-file.rkt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index dfd3024..fdf868b 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -29,16 +29,6 @@ ; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) (define (html-from-url image-url) - ([media-detail (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") - ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . ,prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)] (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) (define content-type (response-headers-ref dest-res "Content-Type")) From 91a3d7255f7559d164dd7a649fc6627e227612b8 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:28:22 +0700 Subject: [PATCH 22/46] easy: --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index fdf868b..99dedee 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -31,7 +31,7 @@ (define (html-from-url image-url) (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) - (define content-type (response-headers-ref dest-res "Content-Type")) + (define content-type (easy:response-headers-ref dest-res "Content-Type")) (cond [(eq? content-type #f) ""] [(regexp-match? #rx"^image/" content-type) From 6d2f5904fe43b79a216282c043596a3e99e0f211 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:36:13 +0700 Subject: [PATCH 23/46] `? --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 99dedee..9be0958 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -32,7 +32,7 @@ (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) (define content-type (easy:response-headers-ref dest-res "Content-Type")) - (cond + `(cond [(eq? content-type #f) ""] [(regexp-match? #rx"^image/" content-type) (img (@ (src ,(u-proxy-url image-url))))] @@ -67,7 +67,7 @@ `(div ,(if (non-empty-string? video-embed-code) (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - `(html-from-url image-url)) + (html-from-url image-url)) (p ,(if (non-empty-string? video-embed-code) `"" From 881b95bde3dc967e65af6764da983c4afeab54f9 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:38:44 +0700 Subject: [PATCH 24/46] ok --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 9be0958..5d4b886 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -31,7 +31,7 @@ (define (html-from-url image-url) (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) - (define content-type (easy:response-headers-ref dest-res "Content-Type")) + (define content-type (easy:response-headers-ref dest-res 'Content-Type)) `(cond [(eq? content-type #f) ""] [(regexp-match? #rx"^image/" content-type) From 85e23c803a205d002b2638b6d52e5062542e271e Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:40:47 +0700 Subject: [PATCH 25/46] ' --- src/page-file.rkt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 5d4b886..c4a1a1f 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -32,14 +32,14 @@ (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) (define content-type (easy:response-headers-ref dest-res 'Content-Type)) - `(cond + (cond [(eq? content-type #f) ""] [(regexp-match? #rx"^image/" content-type) - (img (@ (src ,(u-proxy-url image-url))))] + '(img (@ (src ,(u-proxy-url image-url))))] [(regexp-match? #rx"^audio/" content-type) - (audio (@ (src ,(u-proxy-url image-url)) (controls)))] + '(audio (@ (src ,(u-proxy-url image-url)) (controls)))] [(regexp-match? #rx"^video/" content-type) - (video (@ (src ,(u-proxy-url image-url)) (controls)))] + '(video (@ (src ,(u-proxy-url image-url)) (controls)))] [else ""])) (define (generate-results-page @@ -67,7 +67,7 @@ `(div ,(if (non-empty-string? video-embed-code) (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - (html-from-url image-url)) + `(html-from-url image-url)) (p ,(if (non-empty-string? video-embed-code) `"" From 1c676fd5a0a95e4e03ab9b0db64deea40e25d727 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:41:43 +0700 Subject: [PATCH 26/46] ??? --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index c4a1a1f..2a82def 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -32,7 +32,7 @@ (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) (define content-type (easy:response-headers-ref dest-res 'Content-Type)) - (cond + `(cond [(eq? content-type #f) ""] [(regexp-match? #rx"^image/" content-type) '(img (@ (src ,(u-proxy-url image-url))))] @@ -67,7 +67,7 @@ `(div ,(if (non-empty-string? video-embed-code) (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - `(html-from-url image-url)) + (html-from-url image-url)) (p ,(if (non-empty-string? video-embed-code) `"" From f6b448ae8f1935c0fce7f90c38fd63091499675e Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:42:43 +0700 Subject: [PATCH 27/46] idfk --- src/page-file.rkt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 2a82def..953d04f 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -32,15 +32,15 @@ (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) (define content-type (easy:response-headers-ref dest-res 'Content-Type)) - `(cond - [(eq? content-type #f) ""] + (cond + [(eq? content-type #f) `""] [(regexp-match? #rx"^image/" content-type) - '(img (@ (src ,(u-proxy-url image-url))))] + `(img (@ (src ,(u-proxy-url image-url))))] [(regexp-match? #rx"^audio/" content-type) - '(audio (@ (src ,(u-proxy-url image-url)) (controls)))] + `(audio (@ (src ,(u-proxy-url image-url)) (controls)))] [(regexp-match? #rx"^video/" content-type) - '(video (@ (src ,(u-proxy-url image-url)) (controls)))] - [else ""])) + `(video (@ (src ,(u-proxy-url image-url)) (controls)))] + [else `""])) (define (generate-results-page #:source-url source-url From cb2fb07b2479fe5e22d7253e7ba5c3a8d3851b61 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:46:38 +0700 Subject: [PATCH 28/46] ogg --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 953d04f..d1a6765 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -36,7 +36,7 @@ [(eq? content-type #f) `""] [(regexp-match? #rx"^image/" content-type) `(img (@ (src ,(u-proxy-url image-url))))] - [(regexp-match? #rx"^audio/" content-type) + [(regexp-match? #rx"^audio/|^application/ogg$" content-type) `(audio (@ (src ,(u-proxy-url image-url)) (controls)))] [(regexp-match? #rx"^video/" content-type) `(video (@ (src ,(u-proxy-url image-url)) (controls)))] From 2cd489658af7104d95604fcce7eaeda7f0c46135 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:52:23 +0700 Subject: [PATCH 29/46] prefix --- src/page-file.rkt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d1a6765..6fba971 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -118,7 +118,10 @@ (if (not (jp "/exists" media-detail #f)) (next-dispatcher) (response-handler - (define title (jp "/fileTitle" media-detail)) + (define title (jp "/fileTitle" media-detail "")) + (define title (if (non-empty-string? title) + (format "File:~a" title) + prefixed-title)) (define body (generate-results-page #:source-url source-url #:wikiname wikiname From 915b4a6ad56dae33c9ff12704f9558743e8ea98b Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 22:55:18 +0700 Subject: [PATCH 30/46] f --- src/page-file.rkt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 6fba971..4348740 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -118,9 +118,9 @@ (if (not (jp "/exists" media-detail #f)) (next-dispatcher) (response-handler - (define title (jp "/fileTitle" media-detail "")) - (define title (if (non-empty-string? title) - (format "File:~a" title) + (define file-title (jp "/fileTitle" media-detail "")) + (define title (if (non-empty-string? file-title) + (format "File:~a" file-title) prefixed-title)) (define body (generate-results-page #:source-url source-url From e845cc460b644d40ea1a69e89912de8a2a13fc21 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 23:14:02 +0700 Subject: [PATCH 31/46] content-type --- src/page-file.rkt | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 4348740..7db138d 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -28,18 +28,20 @@ ; (define category-json-data ; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) -(define (html-from-url image-url) +(define (url-content-type url) (log-outgoing image-url) (define dest-res (easy:head image-url #:timeouts timeouts)) - (define content-type (easy:response-headers-ref dest-res 'Content-Type)) + (easy:response-headers-ref dest-res 'Content-Type)) + +(define (get-media-html url content-type) (cond [(eq? content-type #f) `""] [(regexp-match? #rx"^image/" content-type) - `(img (@ (src ,(u-proxy-url image-url))))] + `(img (@ (src ,(u-proxy-url url))))] [(regexp-match? #rx"^audio/|^application/ogg$" content-type) - `(audio (@ (src ,(u-proxy-url image-url)) (controls)))] + `(audio (@ (src ,(u-proxy-url url)) (controls)))] [(regexp-match? #rx"^video/" content-type) - `(video (@ (src ,(u-proxy-url image-url)) (controls)))] + `(video (@ (src ,(u-proxy-url url)) (controls)))] [else `""])) (define (generate-results-page @@ -47,6 +49,7 @@ #:wikiname wikiname #:title title #:media-detail media-detail + #:image-content-type image-content-type #:license [license #f]) (define video-embed-code (jp "/videoEmbedCode" media-detail "")) (define raw-image-url (jp "/rawImageUrl" media-detail)) @@ -67,7 +70,7 @@ `(div ,(if (non-empty-string? video-embed-code) (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - (html-from-url image-url)) + (get-media-html image-url image-content-type)) (p ,(if (non-empty-string? video-embed-code) `"" @@ -122,11 +125,15 @@ (define title (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) + (define image-content-type (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) + #f + (url-content-type (jp "/imageUrl" media-detail)))) (define body (generate-results-page #:source-url source-url #:wikiname wikiname #:title title #:media-detail media-detail + #:image-content-type image-content-type #:license license)) (when (config-true? 'debug) From f2de0cc11f3b7da7d55a76bdbbf3fe5db34be356 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 23:14:57 +0700 Subject: [PATCH 32/46] url --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 7db138d..472652e 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -29,8 +29,8 @@ ; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) (define (url-content-type url) - (log-outgoing image-url) - (define dest-res (easy:head image-url #:timeouts timeouts)) + (log-outgoing url) + (define dest-res (easy:head url #:timeouts timeouts)) (easy:response-headers-ref dest-res 'Content-Type)) (define (get-media-html url content-type) From fb217005172ac4efe3cffe7f12d583cd74cc2c37 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:43:29 +0700 Subject: [PATCH 33/46] shrug --- src/page-file.rkt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 472652e..7fc9bd1 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -31,17 +31,17 @@ (define (url-content-type url) (log-outgoing url) (define dest-res (easy:head url #:timeouts timeouts)) - (easy:response-headers-ref dest-res 'Content-Type)) + (easy:response-headers-ref dest-res 'content-type)) (define (get-media-html url content-type) (cond [(eq? content-type #f) `""] - [(regexp-match? #rx"^image/" content-type) - `(img (@ (src ,(u-proxy-url url))))] - [(regexp-match? #rx"^audio/|^application/ogg$" content-type) - `(audio (@ (src ,(u-proxy-url url)) (controls)))] - [(regexp-match? #rx"^video/" content-type) - `(video (@ (src ,(u-proxy-url url)) (controls)))] + [(regexp-match? #rx"^(?i)image/" content-type) + `(img (@ (src ,url)))] + [(regexp-match? #rx"^(?i)audio/|^(?i)application/ogg(;|$)" content-type) + `(audio (@ (src ,url) (controls)))] + [(regexp-match? #rx"^(?i)video/" content-type) + `(video (@ (src ,url) (controls)))] [else `""])) (define (generate-results-page From 1a996e7c89295d5d1851c7f670f6318e3c91c0c9 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:45:37 +0700 Subject: [PATCH 34/46] odd --- src/page-file.rkt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 7fc9bd1..ef1c2f4 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -36,11 +36,11 @@ (define (get-media-html url content-type) (cond [(eq? content-type #f) `""] - [(regexp-match? #rx"^(?i)image/" content-type) + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,url)))] - [(regexp-match? #rx"^(?i)audio/|^(?i)application/ogg(;|$)" content-type) + [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) `(audio (@ (src ,url) (controls)))] - [(regexp-match? #rx"^(?i)video/" content-type) + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,url) (controls)))] [else `""])) From 51bf087b30b8626038e4799726255510fbecbb33 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:49:10 +0700 Subject: [PATCH 35/46] raco fmt --- src/page-file.rkt | 150 ++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 84 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index ef1c2f4..d04c135 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -20,8 +20,7 @@ "url-utils.rkt" "xexpr-utils.rkt") -(provide - page-file) +(provide page-file) ;(module+ test ; (require rackunit) @@ -36,21 +35,18 @@ (define (get-media-html url content-type) (cond [(eq? content-type #f) `""] - [(regexp-match? #rx"(?i:^image/)" content-type) - `(img (@ (src ,url)))] + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,url)))] [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) - `(audio (@ (src ,url) (controls)))] - [(regexp-match? #rx"(?i:^video/)" content-type) - `(video (@ (src ,url) (controls)))] + `(audio (@ (src ,url) (controls)))] + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,url) (controls)))] [else `""])) -(define (generate-results-page - #:source-url source-url - #:wikiname wikiname - #:title title - #:media-detail media-detail - #:image-content-type image-content-type - #:license [license #f]) +(define (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:license [license #f]) (define video-embed-code (jp "/videoEmbedCode" media-detail "")) (define raw-image-url (jp "/rawImageUrl" media-detail)) (define image-url (jp "/imageUrl" media-detail raw-image-url)) @@ -59,9 +55,8 @@ (define smaller-article-list (jp "/smallerArticleList" media-detail)) (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) (define image-description (jp "/imageDescription" media-detail #f)) - (define maybe-proxied-raw-image-url (if (config-true? 'strict_proxy) - (u-proxy-url raw-image-url) - raw-image-url)) + (define maybe-proxied-raw-image-url + (if (config-true? 'strict_proxy) (u-proxy-url raw-image-url) raw-image-url)) (generate-wiki-page #:source-url source-url #:wikiname wikiname @@ -69,35 +64,27 @@ #:license license `(div ,(if (non-empty-string? video-embed-code) - (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) - (get-media-html image-url image-content-type)) - (p - ,(if (non-empty-string? video-embed-code) - `"" - `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) - "Added by " - (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) - "." - ,(if is-posted-in - `(span " Posted in " - ,@(map - (λ (article) - (define page-path (jp "/title" article)) - (define title (jp "/titleText" article page-path)) - `(span - ,(if (eq? (car smaller-article-list) article) - "" - ", ") - (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) - ,title))) - smaller-article-list) - ,(if (eq? article-list-is-smaller 1) - "…" - ".")) - `"")) + (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) + (get-media-html image-url image-content-type)) + (p ,(if (non-empty-string? video-embed-code) + `"" + `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) + "Added by " + (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) + "." + ,(if is-posted-in + `(span " Posted in " + ,@(map (λ (article) + (define page-path (jp "/title" article)) + (define title (jp "/titleText" article page-path)) + `(span ,(if (eq? (car smaller-article-list) article) "" ", ") + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) ,title))) + smaller-article-list) + ,(if (eq? article-list-is-smaller 1) "…" ".")) + `"")) ,(if (string? image-description) - (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) - "")))) + (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) + "")))) (define (page-file req) (define wikiname (path/param-path (first (url-path (request-uri req))))) @@ -105,45 +92,40 @@ (define origin (format "https://~a.fandom.com" wikiname)) (define source-url (format "~a/wiki/~a" origin prefixed-title)) - (thread-let - ([media-detail (define dest-url - (format "~a/wikia.php?~a" - origin - (params->query `(("format" . "json") - ("controller" . "Lightbox") - ("method" . "getMediaDetail") - ("fileTitle" . ,prefixed-title))))) - (log-outgoing dest-url) - (define dest-res (easy:get dest-url #:timeouts timeouts)) - (easy:response-json dest-res)] - [license (license-auto wikiname)]) - - (if (not (jp "/exists" media-detail #f)) - (next-dispatcher) - (response-handler - (define file-title (jp "/fileTitle" media-detail "")) - (define title (if (non-empty-string? file-title) - (format "File:~a" file-title) - prefixed-title)) - (define image-content-type (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) - #f - (url-content-type (jp "/imageUrl" media-detail)))) - (define body (generate-results-page - #:source-url source-url - #:wikiname wikiname - #:title title - #:media-detail media-detail - #:image-content-type image-content-type - #:license license)) - - (when (config-true? 'debug) - ; used for its side effects - ; convert to string with error checking, error will be raised if xexp is invalid - (xexp->html body)) - (response/output - #:code 200 - (λ (out) - (write-html body out))))))) + (thread-let ([media-detail + (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [license (license-auto wikiname)]) + (if (not (jp "/exists" media-detail #f)) + (next-dispatcher) + (response-handler + (define file-title (jp "/fileTitle" media-detail "")) + (define title + (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) + (define image-content-type + (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) + #f + (url-content-type (jp "/imageUrl" media-detail)))) + (define body + (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:license license)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output #:code 200 + (λ (out) (write-html body out))))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") ; (generate-results-page From 5a59545963d90eecf0715f4b0788d6a189b8971a Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:53:02 +0700 Subject: [PATCH 36/46] Set Referrer-Policy to no-referrer Fandom sends a fake 404 to media if there's a Referer header that has an origin that's not Fandom. However, we can choose not to send the header by setting Referrer-Policy. See also: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy --- src/application-globals.rkt | 3 +++ src/page-category.rkt | 1 + src/page-search.rkt | 1 + src/page-wiki.rkt | 30 +++++++++++++++++------------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index c214924..9a21592 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -9,6 +9,8 @@ "url-utils.rkt") (provide + ; header to not send referers to fandom + referrer-policy ; timeout durations for http-easy requests timeouts ; generates a consistent footer @@ -22,6 +24,7 @@ (require rackunit html-writing)) +(define referrer-policy (header #"Referrer-Policy" #"no-referrer")) (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define (application-footer source-url #:license [license-in #f]) diff --git a/src/page-category.rkt b/src/page-category.rkt index f7c43b2..c9c3ec2 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -113,6 +113,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-search.rkt b/src/page-search.rkt index 387deab..d42fce9 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -81,6 +81,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index a218dfe..be24af8 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -152,15 +152,17 @@ (λ (v) (dict-update v 'rel (λ (s) (list (string-append (car s) " noreferrer"))) '("")))) - ; proxy images from inline styles - (curry attribute-maybe-update 'style - (λ (style) - (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style - (λ (whole url) - (string-append - "url(" - (u-proxy-url url) - ")"))))) + ; proxy images from inline styles, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'style + (λ (style) + (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style + (λ (whole url) + (string-append + "url(" + (u-proxy-url url) + ")")))) v))) ; and also their links, if strict_proxy is set (curry u (λ (v) @@ -168,8 +170,10 @@ (eq? element-type 'a) (has-class? "image-thumbnail" v))) (λ (v) (attribute-maybe-update 'href u-proxy-url v))) - ; proxy images from src attributes - (curry attribute-maybe-update 'src u-proxy-url) + ; proxy images from src attributes, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'src u-proxy-url v))) ; don't lazyload images (curry u (λ (v) (dict-has-key? v 'data-src)) @@ -276,8 +280,8 @@ (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value))) - (list))) + (list (header #"Refresh" value) referrer-policy)) + (list referrer-policy))) (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid From bc07a37bf7ba2551f2f4b2e8d0a20551f3c11ef5 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 10:53:02 +0700 Subject: [PATCH 37/46] Set Referrer-Policy to no-referrer Fandom sends a fake 404 to media if there's a Referer header that has an origin that's not Fandom. However, we can choose not to send the header by setting Referrer-Policy. See also: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy --- src/application-globals.rkt | 3 +++ src/page-category.rkt | 1 + src/page-search.rkt | 1 + src/page-wiki.rkt | 30 +++++++++++++++++------------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/application-globals.rkt b/src/application-globals.rkt index c214924..9a21592 100644 --- a/src/application-globals.rkt +++ b/src/application-globals.rkt @@ -9,6 +9,8 @@ "url-utils.rkt") (provide + ; header to not send referers to fandom + referrer-policy ; timeout durations for http-easy requests timeouts ; generates a consistent footer @@ -22,6 +24,7 @@ (require rackunit html-writing)) +(define referrer-policy (header #"Referrer-Policy" #"no-referrer")) (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define (application-footer source-url #:license [license-in #f]) diff --git a/src/page-category.rkt b/src/page-category.rkt index f7c43b2..c9c3ec2 100644 --- a/src/page-category.rkt +++ b/src/page-category.rkt @@ -113,6 +113,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-search.rkt b/src/page-search.rkt index 387deab..d42fce9 100644 --- a/src/page-search.rkt +++ b/src/page-search.rkt @@ -81,6 +81,7 @@ (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out)))))) (module+ test diff --git a/src/page-wiki.rkt b/src/page-wiki.rkt index a218dfe..be24af8 100644 --- a/src/page-wiki.rkt +++ b/src/page-wiki.rkt @@ -152,15 +152,17 @@ (λ (v) (dict-update v 'rel (λ (s) (list (string-append (car s) " noreferrer"))) '("")))) - ; proxy images from inline styles - (curry attribute-maybe-update 'style - (λ (style) - (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style - (λ (whole url) - (string-append - "url(" - (u-proxy-url url) - ")"))))) + ; proxy images from inline styles, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'style + (λ (style) + (regexp-replace #rx"url\\(['\"]?(.*?)['\"]?\\)" style + (λ (whole url) + (string-append + "url(" + (u-proxy-url url) + ")")))) v))) ; and also their links, if strict_proxy is set (curry u (λ (v) @@ -168,8 +170,10 @@ (eq? element-type 'a) (has-class? "image-thumbnail" v))) (λ (v) (attribute-maybe-update 'href u-proxy-url v))) - ; proxy images from src attributes - (curry attribute-maybe-update 'src u-proxy-url) + ; proxy images from src attributes, if strict_proxy is set + (curry u + (λ (v) (config-true? 'strict_proxy)) + (λ (v) (attribute-maybe-update 'src u-proxy-url v))) ; don't lazyload images (curry u (λ (v) (dict-has-key? v 'data-src)) @@ -276,8 +280,8 @@ (define headers (if redirect-msg (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) - (list (header #"Refresh" value))) - (list))) + (list (header #"Refresh" value) referrer-policy)) + (list referrer-policy))) (when (config-true? 'debug) ; used for its side effects ; convert to string with error checking, error will be raised if xexp is invalid From ba806d0633c5246910df429fd2e4b477decf86e9 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:16:15 +0700 Subject: [PATCH 38/46] simple tests --- src/page-file.rkt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d04c135..a2490ab 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -33,13 +33,22 @@ (easy:response-headers-ref dest-res 'content-type)) (define (get-media-html url content-type) + (define proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) (cond [(eq? content-type #f) `""] - [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,url)))] + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,proxied-url)))] [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) - `(audio (@ (src ,url) (controls)))] - [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,url) (controls)))] + `(audio (@ (src ,proxied-url) (controls)))] + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,proxied-url) (controls)))] [else `""])) +(module+ test + (require rackunit) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") (img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") (video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd")))) + (check-equal? (get-media-html "https://example.com" "who knows") "") + (check-equal? (get-media-html #f "who knows") "")) (define (generate-results-page #:source-url source-url #:wikiname wikiname @@ -125,6 +134,7 @@ ; convert to string with error checking, error will be raised if xexp is invalid (xexp->html body)) (response/output #:code 200 + #:headers (list referrer-policy) (λ (out) (write-html body out))))))) ;(module+ test ; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") From 19aeeea60fc6c7c724206c2ba2d21f847c366d98 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:21:28 +0700 Subject: [PATCH 39/46] more stuff --- src/page-file.rkt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index a2490ab..d20365f 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -33,20 +33,22 @@ (easy:response-headers-ref dest-res 'content-type)) (define (get-media-html url content-type) - (define proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) + (define maybe-proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) (cond [(eq? content-type #f) `""] - [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,proxied-url)))] + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,maybe-proxied-url)))] [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) - `(audio (@ (src ,proxied-url) (controls)))] - [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,proxied-url) (controls)))] + `(audio (@ (src ,maybe-proxied-url) (controls)))] + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,maybe-proxied-url) (controls)))] [else `""])) (module+ test (require rackunit) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") (img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") (video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd")))) + (parameterize ([config-parameter 'strict_proxy "true"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") (img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb"))))) + (parameterize ([config-parameter 'strict_proxy "no"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") (video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd"))))) (check-equal? (get-media-html "https://example.com" "who knows") "") (check-equal? (get-media-html #f "who knows") "")) From 50719142bc5f0d02ee272ffc13f011ed528af798 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:22:23 +0700 Subject: [PATCH 40/46] fix syntax --- src/page-file.rkt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index d20365f..c33b7b9 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -43,10 +43,10 @@ [else `""])) (module+ test (require rackunit) - (parameterize ([config-parameter 'strict_proxy "true"]) + (parameterize ([(config-parameter 'strict_proxy) "true"]) (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") (img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb"))))) - (parameterize ([config-parameter 'strict_proxy "no"]) + (parameterize ([(config-parameter 'strict_proxy) "no"]) (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") (video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd"))))) (check-equal? (get-media-html "https://example.com" "who knows") "") From 99be807b73a9204ec4c98562c66229a53f38eedb Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:23:07 +0700 Subject: [PATCH 41/46] more ` --- src/page-file.rkt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index c33b7b9..f51f945 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -44,13 +44,13 @@ (module+ test (require rackunit) (parameterize ([(config-parameter 'strict_proxy) "true"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") (img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb"))))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb"))))) (parameterize ([(config-parameter 'strict_proxy) "no"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") (audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") (video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd"))))) - (check-equal? (get-media-html "https://example.com" "who knows") "") - (check-equal? (get-media-html #f "who knows") "")) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd"))))) + (check-equal? (get-media-html "https://example.com" "who knows") `"") + (check-equal? (get-media-html #f "who knows") `"")) (define (generate-results-page #:source-url source-url #:wikiname wikiname From bab6c562059ee158eafd91c6d2bc9ffd7381ebcb Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:25:08 +0700 Subject: [PATCH 42/46] oops --- src/page-file.rkt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index f51f945..7e04cd2 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -45,12 +45,12 @@ (require rackunit) (parameterize ([(config-parameter 'strict_proxy) "true"]) (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb"))))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") (controls))))) (parameterize ([(config-parameter 'strict_proxy) "no"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd"))))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") (controls)))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") (controls))))) (check-equal? (get-media-html "https://example.com" "who knows") `"") - (check-equal? (get-media-html #f "who knows") `"")) + (check-equal? (get-media-html "https://example.com" #f) `"")) (define (generate-results-page #:source-url source-url #:wikiname wikiname From 96af163773f9c919ab86d9c04d9947eb1e314e13 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:25:59 +0700 Subject: [PATCH 43/46] fmt --- src/page-file.rkt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 7e04cd2..76134e4 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -44,11 +44,18 @@ (module+ test (require rackunit) (parameterize ([(config-parameter 'strict_proxy) "true"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") (controls))))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") + `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") + (controls))))) (parameterize ([(config-parameter 'strict_proxy) "no"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") (controls)))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") (controls))))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") + (controls)))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") + `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") + (controls))))) (check-equal? (get-media-html "https://example.com" "who knows") `"") (check-equal? (get-media-html "https://example.com" #f) `"")) From c3179e71dcf68f4ac3790f75e0723482f992453e Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:45:36 +0700 Subject: [PATCH 44/46] more test --- src/page-file.rkt | 58 +++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 76134e4..4905c76 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -22,10 +22,10 @@ (provide page-file) -;(module+ test -; (require rackunit) -; (define category-json-data -; '#hasheq((batchcomplete . #t) (continue . #hasheq((cmcontinue . "page|4150504c45|41473") (continue . "-||"))) (query . #hasheq((categorymembers . (#hasheq((ns . 0) (pageid . 25049) (title . "Item (entity)")) #hasheq((ns . 0) (pageid . 128911) (title . "3D")) #hasheq((ns . 0) (pageid . 124018) (title . "A Very Fine Item")) #hasheq((ns . 0) (pageid . 142208) (title . "Amethyst Shard")) #hasheq((ns . 0) (pageid . 121612) (title . "Ankle Monitor"))))))))) +(module+ test + (require rackunit) + (define test-media-detail + '#hasheq((fileTitle . "Example file") (videoEmbedCode . "") (imageUrl . "https://static.wikia.nocookie.net/examplefile") (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") (userName . "blankie") (isPostedIn . #t) (smallerArticleList . (list #hasheq((title . "Example_article") (titleText . "Example article")))) (articleListIsSmaller . 0) (exists . #t) (imageDescription . #f)))) (define (url-content-type url) (log-outgoing url) @@ -41,23 +41,6 @@ `(audio (@ (src ,maybe-proxied-url) (controls)))] [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,maybe-proxied-url) (controls)))] [else `""])) -(module+ test - (require rackunit) - (parameterize ([(config-parameter 'strict_proxy) "true"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") - `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") - `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") - (controls))))) - (parameterize ([(config-parameter 'strict_proxy) "no"]) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") - `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") - (controls)))) - (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") - `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") - (controls))))) - (check-equal? (get-media-html "https://example.com" "who knows") `"") - (check-equal? (get-media-html "https://example.com" #f) `"")) (define (generate-results-page #:source-url source-url #:wikiname wikiname @@ -145,11 +128,28 @@ (response/output #:code 200 #:headers (list referrer-policy) (λ (out) (write-html body out))))))) -;(module+ test -; (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor") -; (generate-results-page -; #:source-url "" -; #:wikiname "test" -; #:title "Category:Items" -; #:members-data category-json-data -; #:page '(div "page text")))))) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") + `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") + (controls))))) + (parameterize ([(config-parameter 'strict_proxy) "no"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") + (controls)))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") + `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") + (controls))))) + (check-equal? (get-media-html "https://example.com" "who knows") `"") + (check-equal? (get-media-html "https://example.com" #f) `"")) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-not-false ((query-selector (attribute-selector 'src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fexamplefile") + (generate-results-page + #:source-url "" + #:wikiname "test" + #:title "File:Example file" + #:media-detail test-media-detail + #:image-content-type "image/jpeg")))))) From 0d03783bbf960c85b148d2e9abddf30a6c5ad325 Mon Sep 17 00:00:00 2001 From: blankie Date: Sun, 9 Oct 2022 14:49:46 +0700 Subject: [PATCH 45/46] () --- src/page-file.rkt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page-file.rkt b/src/page-file.rkt index 4905c76..c7018b2 100644 --- a/src/page-file.rkt +++ b/src/page-file.rkt @@ -25,7 +25,7 @@ (module+ test (require rackunit) (define test-media-detail - '#hasheq((fileTitle . "Example file") (videoEmbedCode . "") (imageUrl . "https://static.wikia.nocookie.net/examplefile") (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") (userName . "blankie") (isPostedIn . #t) (smallerArticleList . (list #hasheq((title . "Example_article") (titleText . "Example article")))) (articleListIsSmaller . 0) (exists . #t) (imageDescription . #f)))) + '#hasheq((fileTitle . "Example file") (videoEmbedCode . "") (imageUrl . "https://static.wikia.nocookie.net/examplefile") (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") (userName . "blankie") (isPostedIn . #t) (smallerArticleList . (#hasheq((title . "Example_article") (titleText . "Example article")))) (articleListIsSmaller . 0) (exists . #t) (imageDescription . #f)))) (define (url-content-type url) (log-outgoing url) From bf80692c4e6338dad466eb5727eb00f65ca94a52 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 8 Oct 2022 15:35:35 +0700 Subject: [PATCH 46/46] Add support for File: pages Fixes https://lists.sr.ht/~cadence/breezewiki-discuss/%3Cb2835a70-5118-4df0-90c9-4333486a4b69%40nixnetmail.com%3E --- breezewiki.rkt | 2 + dist.rkt | 2 + src/dispatcher-tree.rkt | 1 + src/page-file.rkt | 166 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 src/page-file.rkt diff --git a/breezewiki.rkt b/breezewiki.rkt index dfb405e..3fc9b8f 100644 --- a/breezewiki.rkt +++ b/breezewiki.rkt @@ -19,6 +19,7 @@ (require-reloadable "src/page-static.rkt" static-dispatcher) (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher) (require-reloadable "src/page-wiki.rkt" page-wiki) +(require-reloadable "src/page-file.rkt" page-file) (reload!) @@ -38,6 +39,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher)))) diff --git a/dist.rkt b/dist.rkt index a626695..805df48 100644 --- a/dist.rkt +++ b/dist.rkt @@ -13,6 +13,7 @@ (require (only-in "src/page-static.rkt" static-dispatcher)) (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher)) (require (only-in "src/page-wiki.rkt" page-wiki)) +(require (only-in "src/page-file.rkt" page-file)) (serve/launch/wait #:listen-ip (if (config-true? 'debug) "127.0.0.1" #f) @@ -27,6 +28,7 @@ page-proxy page-search page-wiki + page-file redirect-wiki-home static-dispatcher subdomain-dispatcher))) diff --git a/src/dispatcher-tree.rkt b/src/dispatcher-tree.rkt index f2c1412..b68cf9c 100644 --- a/src/dispatcher-tree.rkt +++ b/src/dispatcher-tree.rkt @@ -45,6 +45,7 @@ (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) + (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) diff --git a/src/page-file.rkt b/src/page-file.rkt new file mode 100644 index 0000000..7635eb4 --- /dev/null +++ b/src/page-file.rkt @@ -0,0 +1,166 @@ +#lang racket/base +(require racket/dict + racket/list + racket/match + racket/string + (prefix-in easy: net/http-easy) + ; html libs + html-parsing + html-writing + ; web server libs + net/url + web-server/http + (only-in web-server/dispatchers/dispatch next-dispatcher) + #;(only-in web-server/http/redirect redirect-to) + "application-globals.rkt" + "config.rkt" + "data.rkt" + "page-wiki.rkt" + "syntax.rkt" + "url-utils.rkt" + "xexpr-utils.rkt") + +(provide page-file) + +(module+ test + (require rackunit) + (define test-media-detail + '#hasheq((fileTitle . "Example file") + (videoEmbedCode . "") + (imageUrl . "https://static.wikia.nocookie.net/examplefile") + (rawImageUrl . "https://static.wikia.nocookie.net/examplefile") + (userName . "blankie") + (isPostedIn . #t) + (smallerArticleList . (#hasheq((title . "Example_article") + (titleText . "Example article")))) + (articleListIsSmaller . 0) + (exists . #t) + (imageDescription . #f)))) + +(define (url-content-type url) + (log-outgoing url) + (define dest-res (easy:head url #:timeouts timeouts)) + (easy:response-headers-ref dest-res 'content-type)) + +(define (get-media-html url content-type) + (define maybe-proxied-url (if (config-true? 'strict_proxy) (u-proxy-url url) url)) + (cond + [(eq? content-type #f) `""] + [(regexp-match? #rx"(?i:^image/)" content-type) `(img (@ (src ,maybe-proxied-url)))] + [(regexp-match? #rx"(?i:^audio/|^application/ogg(;|$))" content-type) + `(audio (@ (src ,maybe-proxied-url) (controls)))] + [(regexp-match? #rx"(?i:^video/)" content-type) `(video (@ (src ,maybe-proxied-url) (controls)))] + [else `""])) + +(define (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:license [license #f]) + (define video-embed-code (jp "/videoEmbedCode" media-detail "")) + (define raw-image-url (jp "/rawImageUrl" media-detail)) + (define image-url (jp "/imageUrl" media-detail raw-image-url)) + (define username (jp "/userName" media-detail)) + (define is-posted-in (jp "/isPostedIn" media-detail #f)) + (define smaller-article-list (jp "/smallerArticleList" media-detail)) + (define article-list-is-smaller (jp "/articleListIsSmaller" media-detail)) + (define image-description (jp "/imageDescription" media-detail #f)) + (define maybe-proxied-raw-image-url + (if (config-true? 'strict_proxy) (u-proxy-url raw-image-url) raw-image-url)) + (generate-wiki-page + #:source-url source-url + #:wikiname wikiname + #:title title + #:license license + `(div ,(if (non-empty-string? video-embed-code) + (update-tree-wiki (html->xexp (preprocess-html-wiki video-embed-code)) wikiname) + (get-media-html image-url image-content-type)) + (p ,(if (non-empty-string? video-embed-code) + `"" + `(span (a (@ (href ,maybe-proxied-raw-image-url)) "View original file") ". ")) + "Added by " + (a (@ (href ,(format "/~a/wiki/User:~a" wikiname username))) ,username) + "." + ,(if is-posted-in + `(span " Posted in " + ,@(map (λ (article) + (define page-path (jp "/title" article)) + (define title (jp "/titleText" article page-path)) + `(span ,(if (eq? (car smaller-article-list) article) "" ", ") + (a (@ (href ,(format "/~a/wiki/~a" wikiname page-path))) + ,title))) + smaller-article-list) + ,(if (eq? article-list-is-smaller 1) "…" ".")) + `"")) + ,(if (string? image-description) + (update-tree-wiki (html->xexp (preprocess-html-wiki image-description)) wikiname) + "")))) + +(define (page-file req) + (define wikiname (path/param-path (first (url-path (request-uri req))))) + (define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) + (define origin (format "https://~a.fandom.com" wikiname)) + (define source-url (format "~a/wiki/~a" origin prefixed-title)) + + (thread-let ([media-detail + (define dest-url + (format "~a/wikia.php?~a" + origin + (params->query `(("format" . "json") ("controller" . "Lightbox") + ("method" . "getMediaDetail") + ("fileTitle" . ,prefixed-title))))) + (log-outgoing dest-url) + (define dest-res (easy:get dest-url #:timeouts timeouts)) + (easy:response-json dest-res)] + [license (license-auto wikiname)]) + (if (not (jp "/exists" media-detail #f)) + (next-dispatcher) + (response-handler + (define file-title (jp "/fileTitle" media-detail "")) + (define title + (if (non-empty-string? file-title) (format "File:~a" file-title) prefixed-title)) + (define image-content-type + (if (non-empty-string? (jp "/videoEmbedCode" media-detail "")) + #f + (url-content-type (jp "/imageUrl" media-detail)))) + (define body + (generate-results-page #:source-url source-url + #:wikiname wikiname + #:title title + #:media-detail media-detail + #:image-content-type image-content-type + #:license license)) + (when (config-true? 'debug) + ; used for its side effects + ; convert to string with error checking, error will be raised if xexp is invalid + (xexp->html body)) + (response/output #:code 200 + #:headers (list referrer-policy) + (λ (out) (write-html body out))))))) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/a" "image/jpeg") + `(img (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fa")))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/b" "audio/mp3") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fb") + (controls))))) + (parameterize ([(config-parameter 'strict_proxy) "no"]) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/c" "application/ogg") + `(audio (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fc") + (controls)))) + (check-equal? (get-media-html "https://static.wikia.nocookie.net/d" "video/mp4") + `(video (@ (src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fd") + (controls))))) + (check-equal? (get-media-html "https://example.com" "who knows") `"") + (check-equal? (get-media-html "https://example.com" #f) `"")) +(module+ test + (parameterize ([(config-parameter 'strict_proxy) "true"]) + (check-not-false + ((query-selector + (attribute-selector 'src "/proxy?dest=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fexamplefile") + (generate-results-page #:source-url "" + #:wikiname "test" + #:title "File:Example file" + #:media-detail test-media-detail + #:image-content-type "image/jpeg"))))))