Refactor siteinfo/license fetching

info.rkt

@@ -1,3 +1,3 @@
 #lang info
 
-(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib"))
+(define build-deps '("rackunit-lib" "web-server-lib" "http-easy-lib" "html-parsing" "html-writing" "json-pointer" "ini-lib" "memo"))

src/data.rkt

@@ -1,33 +1,34 @@
 #lang racket/base
-(require (prefix-in easy: net/http-easy)
+(require racket/list
+         (prefix-in easy: net/http-easy)
+         memo
          "url-utils.rkt"
          "xexpr-utils.rkt")
 
 (provide
+ (struct-out siteinfo)
  (struct-out license)
- license-default
+ siteinfo-fetch
- license-auto)
+ license-default)
 
+(struct siteinfo (sitename basepage license) #:transparent)
 (struct license (text url) #:transparent)
+
 (define license-default (license "CC-BY-SA" "https://www.fandom.com/licensing"))
-(define license-hash (make-hash))
+
-(define (license-fetch wikiname)
+(define/memoize (siteinfo-fetch wikiname) #:hash hash
   (define dest-url
     (format "https://~a.fandom.com/api.php?~a"
             wikiname
             (params->query '(("action" . "query")
                              ("meta" . "siteinfo")
-                             ("siprop" . "rightsinfo")
+                             ("siprop" . "general|rightsinfo")
                              ("format" . "json")
                              ("formatversion" . "2")))))
   (log-outgoing dest-url)
   (define res (easy:get dest-url))
   (define data (easy:response-json res))
-  (license (jp "/query/rightsinfo/text" data)
+  (siteinfo (jp "/query/general/sitename" data)
-           (jp "/query/rightsinfo/url" data)))
+            (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
-(define (license-auto wikiname)
+            (license (jp "/query/rightsinfo/text" data)
-  (if (hash-has-key? license-hash wikiname)
+                     (jp "/query/rightsinfo/url" data))))
-      (hash-ref license-hash wikiname)
-      (let ([result (license-fetch wikiname)])
-        (hash-set! license-hash wikiname result)
-        result)))

src/page-category.rkt

@@ -89,7 +89,7 @@
                 (log-outgoing dest-url)
                 (define dest-res (easy:get dest-url #:timeouts timeouts))
                 (easy:response-json dest-res)]
-     [license (license-auto wikiname)])
+     [license (siteinfo-license (siteinfo-fetch wikiname))])
 
     (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category)))
     (define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))

src/page-search.rkt

@@ -70,7 +70,7 @@
    (thread-let
     ([dest-res (log-outgoing dest-url)
                (easy:get dest-url #:timeouts timeouts)]
-     [license (license-auto wikiname)])
+     [license (siteinfo-license (siteinfo-fetch wikiname))])
 
     (define data (easy:response-json dest-res))
 

src/page-wiki.rkt

@@ -248,7 +248,7 @@
                                          ("format" . "json")))))
               (log-outgoing dest-url)
               (easy:get dest-url #:timeouts timeouts)]
-    [license (license-auto wikiname)])
+    [license (siteinfo-license (siteinfo-fetch wikiname))])
 
    (cond
      [(eq? 200 (easy:response-status-code dest-res))