forked from cadence/breezewiki
Refactor siteinfo/license fetching
This commit is contained in:
parent
6b176e3f8f
commit
9aba3ad432
5 changed files with 19 additions and 18 deletions
2
info.rkt
2
info.rkt
|
@ -1,3 +1,3 @@
|
|||
#lang info
|
||||
|
||||
(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib"))
|
||||
(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib" "memo"))
|
||||
|
|
27
src/data.rkt
27
src/data.rkt
|
@ -1,33 +1,34 @@
|
|||
#lang racket/base
|
||||
(require (prefix-in easy: net/http-easy)
|
||||
(require racket/list
|
||||
(prefix-in easy: net/http-easy)
|
||||
memo
|
||||
"url-utils.rkt"
|
||||
"xexpr-utils.rkt")
|
||||
|
||||
(provide
|
||||
(struct-out siteinfo)
|
||||
(struct-out license)
|
||||
license-default
|
||||
license-auto)
|
||||
siteinfo-fetch
|
||||
license-default)
|
||||
|
||||
(struct siteinfo (sitename basepage license) #:transparent)
|
||||
(struct license (text url) #:transparent)
|
||||
|
||||
(define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing"))
|
||||
(define license-hash (make-hash))
|
||||
(define (license-fetch wikiname)
|
||||
|
||||
(define/memoize (siteinfo-fetch wikiname) #:hash hash
|
||||
(define dest-url
|
||||
(format "https://~a.fandom.com/api.php?~a"
|
||||
wikiname
|
||||
(params->query '(("action" . "query")
|
||||
("meta" . "siteinfo")
|
||||
("siprop" . "rightsinfo")
|
||||
("siprop" . "general|rightsinfo")
|
||||
("format" . "json")
|
||||
("formatversion" . "2")))))
|
||||
(log-outgoing dest-url)
|
||||
(define res (easy:get dest-url))
|
||||
(define data (easy:response-json res))
|
||||
(siteinfo (jp "/query/general/sitename" data)
|
||||
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
|
||||
(license (jp "/query/rightsinfo/text" data)
|
||||
(jp "/query/rightsinfo/url" data)))
|
||||
(define (license-auto wikiname)
|
||||
(if (hash-has-key? license-hash wikiname)
|
||||
(hash-ref license-hash wikiname)
|
||||
(let ([result (license-fetch wikiname)])
|
||||
(hash-set! license-hash wikiname result)
|
||||
result)))
|
||||
(jp "/query/rightsinfo/url" data))))
|
||||
|
|
|
@ -89,7 +89,7 @@
|
|||
(log-outgoing dest-url)
|
||||
(define dest-res (easy:get dest-url #:timeouts timeouts))
|
||||
(easy:response-json dest-res)]
|
||||
[license (license-auto wikiname)])
|
||||
[license (siteinfo-license (siteinfo-fetch wikiname))])
|
||||
|
||||
(define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category)))
|
||||
(define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))
|
||||
|
|
|
@ -70,7 +70,7 @@
|
|||
(thread-let
|
||||
([dest-res (log-outgoing dest-url)
|
||||
(easy:get dest-url #:timeouts timeouts)]
|
||||
[license (license-auto wikiname)])
|
||||
[license (siteinfo-license (siteinfo-fetch wikiname))])
|
||||
|
||||
(define data (easy:response-json dest-res))
|
||||
|
||||
|
|
|
@ -248,7 +248,7 @@
|
|||
("format" . "json")))))
|
||||
(log-outgoing dest-url)
|
||||
(easy:get dest-url #:timeouts timeouts)]
|
||||
[license (license-auto wikiname)])
|
||||
[license (siteinfo-license (siteinfo-fetch wikiname))])
|
||||
|
||||
(cond
|
||||
[(eq? 200 (easy:response-status-code dest-res))
|
||||
|
|
Loading…
Reference in a new issue