Evalprime/breezewiki
1
0
Fork 0
forked from cadence/breezewiki
Code Pull requests Activity

Display correct license information

Browse source
This commit is contained in:
Cadence Ember 2022-09-17 01:56:03 +12:00
parent 37318b8c50
commit a9acfc34a2
6 changed files with 134 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/application-globals.rkt
View file

@ -4,6 +4,7 @@
html-writing html-writing
web-server/http web-server/http
"config.rkt" "config.rkt"
"data.rkt"
"xexpr-utils.rkt" "xexpr-utils.rkt"
"url-utils.rkt") "url-utils.rkt")
@ -23,7 +24,8 @@
(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5)) (define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
(define (application-footer source-url) (define (application-footer source-url #:license [license-in #f])
(define license (or license-in license-default))
`(footer (@ (class "custom-footer")) `(footer (@ (class "custom-footer"))
(div (@ (class ,(if source-url "custom-footer__cols" "internal-footer"))) (div (@ (class ,(if source-url "custom-footer__cols" "internal-footer")))
(div (p (div (p
@ -46,8 +48,8 @@
,(if source-url ,(if source-url
`(div (p "This page displays proxied content from " `(div (p "This page displays proxied content from "
(a (@ (href ,source-url) (rel "noreferrer")) ,source-url) (a (@ (href ,source-url) (rel "noreferrer")) ,source-url)
". Text content is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " ,(format ". Text content is available under the ~a license, " (license-text license))
(a (@ (href "https://www.fandom.com/licensing")) "see license info.") (a (@ (href ,(license-url license))) "see license info.")
" Media files may have different copying restrictions.") " Media files may have different copying restrictions.")
(p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name)))) (p ,(format "Fandom is a trademark of Fandom, Inc. ~a is not affiliated with Fandom." (config-get 'application_name))))
`(div (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), " `(div (p "Text content on wikis run by Fandom is available under the Creative Commons Attribution-Share Alike License 3.0 (Unported), "
@ -60,8 +62,9 @@
#:source-url source-url #:source-url source-url
#:wikiname wikiname #:wikiname wikiname
#:title title #:title title
#:body-class [body-class-in ""]) #:body-class [body-class-in #f]
(define body-class (if (equal? "" body-class-in) #:license [license #f])
(define body-class (if (not body-class-in)
"skin-fandomdesktop" "skin-fandomdesktop"
body-class-in)) body-class-in))
(define (required-styles origin) (define (required-styles origin)
@ -98,7 +101,7 @@
(div (@ (id "content") #;(class "page-content")) (div (@ (id "content") #;(class "page-content"))
(div (@ (id "mw-content-text")) (div (@ (id "mw-content-text"))
,content)) ,content))
,(application-footer source-url))))))) ,(application-footer source-url #:license license)))))))
(module+ test (module+ test
(define page (define page
(parameterize ([(config-parameter 'strict_proxy) "true"]) (parameterize ([(config-parameter 'strict_proxy) "true"])

33
src/data.rkt Normal file
View file

@ -0,0 +1,33 @@
#lang racket/base
(require (prefix-in easy: net/http-easy)
"url-utils.rkt"
"xexpr-utils.rkt")
(provide
(struct-out license)
license-default
license-auto)
(struct license (text url) #:transparent)
(define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing"))
(define license-hash (make-hash))
(define (license-fetch wikiname)
(define dest-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query '(("action" . "query")
("meta" . "siteinfo")
("siprop" . "rightsinfo")
("format" . "json")
("formatversion" . "2")))))
(printf "out: ~a~n" dest-url)
(define res (easy:get dest-url))
(define data (easy:response-json res))
(license (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))
(define (license-auto wikiname)
(if (hash-has-key? license-hash wikiname)
(hash-ref license-hash wikiname)
(let ([result (license-fetch wikiname)])
(hash-set! license-hash wikiname result)
result)))

4
src/dispatcher-tree.rkt
View file

@ -12,7 +12,9 @@
(provide (provide
; syntax to make the hashmap from names ; syntax to make the hashmap from names
dispatcher-tree) dispatcher-tree
; procedure to make the tree from the hashmap
make-dispatcher-tree)
; make a hashmap out of the provided names and call make-dispatcher-tree with it ; make a hashmap out of the provided names and call make-dispatcher-tree with it
(define-syntax (dispatcher-tree stx) (define-syntax (dispatcher-tree stx)

15
src/page-category.rkt
View file

@ -14,6 +14,7 @@
#;(only-in web-server/http/redirect redirect-to) #;(only-in web-server/http/redirect redirect-to)
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt"
"page-wiki.rkt" "page-wiki.rkt"
"syntax.rkt" "syntax.rkt"
"url-utils.rkt" "url-utils.rkt"
@ -33,13 +34,15 @@
#:prefixed-category prefixed-category #:prefixed-category prefixed-category
#:members-data members-data #:members-data members-data
#:page page #:page page
#:body-class body-class) #:body-class [body-class #f]
#:license [license #f])
(define members (jp "/query/categorymembers" members-data)) (define members (jp "/query/categorymembers" members-data))
(generate-wiki-page (generate-wiki-page
#:source-url source-url #:source-url source-url
#:wikiname wikiname #:wikiname wikiname
#:title prefixed-category #:title prefixed-category
#:body-class body-class #:body-class body-class
#:license license
`(div `(div
,(update-tree-wiki page wikiname) ,(update-tree-wiki page wikiname)
(hr) (hr)
@ -85,7 +88,8 @@
("format" . "json"))))) ("format" . "json")))))
(printf "out: ~a~n" dest-url) (printf "out: ~a~n" dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts)) (define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res)]) (easy:response-json dest-res)]
[license (license-auto wikiname)])
(define page-html (preprocess-html-wiki (jp "/parse/text" page-data ""))) (define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))
(define page (html->xexp page-html)) (define page (html->xexp page-html))
@ -93,15 +97,14 @@
(define body-class (match (regexp-match #rx"<body [^>]*class=\"([^\"]*)" head-html) (define body-class (match (regexp-match #rx"<body [^>]*class=\"([^\"]*)" head-html)
[(list _ classes) classes] [(list _ classes) classes]
[_ ""])) [_ ""]))
(println head-html)
(println body-class)
(define body (generate-results-page (define body (generate-results-page
#:source-url source-url #:source-url source-url
#:wikiname wikiname #:wikiname wikiname
#:prefixed-category prefixed-category #:prefixed-category prefixed-category
#:members-data members-data #:members-data members-data
#:page page #:page page
#:body-class body-class)) #:body-class body-class
#:license license))
(when (config-true? 'debug) (when (config-true? 'debug)
; used for its side effects ; used for its side effects
@ -117,5 +120,5 @@
#:source-url "" #:source-url ""
#:wikiname "test" #:wikiname "test"
#:prefixed-category "Category:Items" #:prefixed-category "Category:Items"
#:category-data category-json-data #:members-data category-json-data
#:page '(div "page text")))))) #:page '(div "page text"))))))

50
src/page-search.rkt
View file

@ -10,8 +10,10 @@
web-server/http web-server/http
(only-in web-server/dispatchers/dispatch next-dispatcher) (only-in web-server/dispatchers/dispatch next-dispatcher)
#;(only-in web-server/http/redirect redirect-to) #;(only-in web-server/http/redirect redirect-to)
"config.rkt"
"application-globals.rkt" "application-globals.rkt"
"config.rkt"
"data.rkt"
"syntax.rkt"
"url-utils.rkt" "url-utils.rkt"
"xexpr-utils.rkt") "xexpr-utils.rkt")
@ -23,12 +25,13 @@
(define search-json-data (define search-json-data
'#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) '#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))))))
(define (generate-results-page dest-url wikiname query data) (define (generate-results-page dest-url wikiname query data #:license [license #f])
(define search-results (jp "/query/search" data)) (define search-results (jp "/query/search" data))
(generate-wiki-page (generate-wiki-page
#:source-url dest-url #:source-url dest-url
#:wikiname wikiname #:wikiname wikiname
#:title "Search Results" #:title "Search Results"
#:license license
`(div (@ (class "mw-parser-output")) `(div (@ (class "mw-parser-output"))
(p ,(format "~a results found for " (length search-results)) (p ,(format "~a results found for " (length search-results))
(strong ,query)) (strong ,query))
@ -54,29 +57,32 @@
(response-handler (response-handler
(define wikiname (path/param-path (first (url-path (request-uri req))))) (define wikiname (path/param-path (first (url-path (request-uri req)))))
(define query (dict-ref (url-query (request-uri req)) 'q #f)) (define query (dict-ref (url-query (request-uri req)) 'q #f))
(define origin (format "https://~a.fandom.com" wikiname)) (define origin (format "https://~a.fandom.com" wikiname))
(define dest-url (format "~a/api.php?~a" (define dest-url
origin (format "~a/api.php?~a"
(params->query `(("action" . "query") origin
("list" . "search") (params->query `(("action" . "query")
("srsearch" . ,query) ("list" . "search")
("formatversion" . "2") ("srsearch" . ,query)
("format" . "json"))))) ("formatversion" . "2")
(printf "out: ~a~n" dest-url) ("format" . "json")))))
(define dest-res (easy:get dest-url #:timeouts timeouts))
(define data (easy:response-json dest-res)) (thread-let
([dest-res (printf "out: ~a~n" dest-url)
(easy:get dest-url #:timeouts timeouts)]
[license (license-auto wikiname)])
(define body (generate-results-page dest-url wikiname query data)) (define data (easy:response-json dest-res))
(when (config-true? 'debug)
; used for its side effects (define body (generate-results-page dest-url wikiname query data #:license license))
; convert to string with error checking, error will be raised if xexp is invalid (when (config-true? 'debug)
(xexp->html body)) ; used for its side effects
(response/output ; convert to string with error checking, error will be raised if xexp is invalid
#:code 200 (xexp->html body))
(λ (out) (response/output
(write-html body out))))) #:code 200
(λ (out)
(write-html body out))))))
(module+ test (module+ test
(check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")
(generate-results-page "" "test" "Gacha" search-json-data))))) (generate-results-page "" "test" "Gacha" search-json-data)))))

98
src/page-wiki.rkt
View file

@ -16,7 +16,9 @@
; my libs ; my libs
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt"
"pure-utils.rkt" "pure-utils.rkt"
"syntax.rkt"
"xexpr-utils.rkt" "xexpr-utils.rkt"
"url-utils.rkt") "url-utils.rkt")
@ -148,7 +150,7 @@
(λ (v) (and (eq? element-type 'a) (λ (v) (and (eq? element-type 'a)
(has-class? "image" v))) (has-class? "image" v)))
(λ (v) (dict-update v 'rel (λ (s) (λ (v) (dict-update v 'rel (λ (s)
(list (string-append (car s) " noreferrer"))) (list (string-append (car s) " noreferrer")))
'("")))) '(""))))
; proxy images from inline styles ; proxy images from inline styles
(curry attribute-maybe-update 'style (curry attribute-maybe-update 'style
@ -235,49 +237,53 @@
(define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define dest-url (format "~a/api.php?~a" (thread-let
origin ([dest-res (define dest-url
(params->query `(("action" . "parse") (format "~a/api.php?~a"
("page" . ,path) origin
("prop" . "text|headhtml|langlinks") (params->query `(("action" . "parse")
("formatversion" . "2") ("page" . ,path)
("format" . "json"))))) ("prop" . "text|headhtml|langlinks")
(printf "out: ~a~n" dest-url) ("formatversion" . "2")
(define dest-res (easy:get dest-url #:timeouts timeouts)) ("format" . "json")))))
(printf "out: ~a~n" dest-url)
(easy:get dest-url #:timeouts timeouts)]
[license (license-auto wikiname)])
(cond (cond
[(eq? 200 (easy:response-status-code dest-res)) [(eq? 200 (easy:response-status-code dest-res))
(let* ([data (easy:response-json dest-res)] (let* ([data (easy:response-json dest-res)]
[title (jp "/parse/title" data "")] [title (jp "/parse/title" data "")]
[page-html (jp "/parse/text" data "")] [page-html (jp "/parse/text" data "")]
[page-html (preprocess-html-wiki page-html)] [page-html (preprocess-html-wiki page-html)]
[page (html->xexp page-html)] [page (html->xexp page-html)]
[head-html (jp "/parse/headhtml" data "")] [head-html (jp "/parse/headhtml" data "")]
[body-class (match (regexp-match #rx"<body [^>]*class=\"([^\"]*)" head-html) [body-class (match (regexp-match #rx"<body [^>]*class=\"([^\"]*)" head-html)
[(list _ classes) classes] [(list _ classes) classes]
[_ ""])]) [_ ""])])
(if (equal? "missingtitle" (jp "/error/code" data #f)) (if (equal? "missingtitle" (jp "/error/code" data #f))
(next-dispatcher) (next-dispatcher)
(response-handler (response-handler
(define body (define body
(generate-wiki-page (generate-wiki-page
(update-tree-wiki page wikiname) (update-tree-wiki page wikiname)
#:source-url source-url #:source-url source-url
#:wikiname wikiname #:wikiname wikiname
#:title title #:title title
#:body-class body-class)) #:body-class body-class
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body))) #:license license))
(define headers (if redirect-msg (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
(let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))] (define headers (if redirect-msg
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))]) (let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))]
(list (header #"Refresh" value))) [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
(list))) (list (header #"Refresh" value)))
(when (config-true? 'debug) (list)))
; used for its side effects (when (config-true? 'debug)
; convert to string with error checking, error will be raised if xexp is invalid ; used for its side effects
(xexp->html body)) ; convert to string with error checking, error will be raised if xexp is invalid
(response/output (xexp->html body))
#:code 200 (response/output
#:headers headers #:code 200
(λ (out) #:headers headers
(write-html body out))))))])) (λ (out)
(write-html body out))))))])))