Refactor siteinfo/license fetching

This commit is contained in:
Cadence Ember 2022-10-09 20:53:09 +13:00
parent 6b176e3f8f
commit 9aba3ad432
Signed by untrusted user: cadence
GPG key ID: BC1C2C61CF521B17
5 changed files with 19 additions and 18 deletions

View file

@ -1,3 +1,3 @@
#lang info
(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib"))
(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib" "memo"))

View file

@ -1,33 +1,34 @@
#lang racket/base
(require (prefix-in easy: net/http-easy)
(require racket/list
(prefix-in easy: net/http-easy)
memo
"url-utils.rkt"
"xexpr-utils.rkt")
(provide
(struct-out siteinfo)
(struct-out license)
license-default
license-auto)
siteinfo-fetch
license-default)
(struct siteinfo (sitename basepage license) #:transparent)
(struct license (text url) #:transparent)
(define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing"))
(define license-hash (make-hash))
(define (license-fetch wikiname)
(define/memoize (siteinfo-fetch wikiname) #:hash hash
(define dest-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query '(("action" . "query")
("meta" . "siteinfo")
("siprop" . "rightsinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2")))))
(log-outgoing dest-url)
(define res (easy:get dest-url))
(define data (easy:response-json res))
(license (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))
(define (license-auto wikiname)
(if (hash-has-key? license-hash wikiname)
(hash-ref license-hash wikiname)
(let ([result (license-fetch wikiname)])
(hash-set! license-hash wikiname result)
result)))
(siteinfo (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data))))

View file

@ -89,7 +89,7 @@
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res)]
[license (license-auto wikiname)])
[license (siteinfo-license (siteinfo-fetch wikiname))])
(define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category)))
(define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))

View file

@ -70,7 +70,7 @@
(thread-let
([dest-res (log-outgoing dest-url)
(easy:get dest-url #:timeouts timeouts)]
[license (license-auto wikiname)])
[license (siteinfo-license (siteinfo-fetch wikiname))])
(define data (easy:response-json dest-res))

View file

@ -248,7 +248,7 @@
("format" . "json")))))
(log-outgoing dest-url)
(easy:get dest-url #:timeouts timeouts)]
[license (license-auto wikiname)])
[license (siteinfo-license (siteinfo-fetch wikiname))])
(cond
[(eq? 200 (easy:response-status-code dest-res))