From bf055836ccb0f7d716ecbe031baea829ebad3f78 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 24 Oct 2022 00:22:47 +1300 Subject: [PATCH] Add download wiki names script --- misc/download-wiki-names.rkt | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 misc/download-wiki-names.rkt diff --git a/misc/download-wiki-names.rkt b/misc/download-wiki-names.rkt new file mode 100644 index 0000000..ff8fff9 --- /dev/null +++ b/misc/download-wiki-names.rkt @@ -0,0 +1,45 @@ +#lang racket/base +(require racket/generator + racket/list + racket/string + json + net/http-easy + html-parsing + "src/xexpr-utils.rkt" + "src/url-utils.rkt") + +(define output-file "wiki-names.json") +(define limit "5000") + +(define (get-page offset) + (define res (get (format "https://community.fandom.com/wiki/Special:NewWikis?~a" + (params->query `(("offset" . ,offset) + ("limit" . ,limit)))))) + (html->xexp (bytes->string/utf-8 (response-body res)))) + +(define (convert-list-items gen) + (for/list ([item (in-producer gen #f)]) + ; '(li "\n" "\t" (a (@ (href "http://terra-hexalis.fandom.com/")) "Terra Hexalis Wiki") "\n" "\t\t\ten\t") + (hasheq 'title (third (fourth item)) + 'link (second (second (second (fourth item)))) + 'lang (string-trim (sixth item))))) + +(define (get-items-recursive [offset ""] [items null]) + (define page (get-page offset)) + (define page-content ((query-selector (attribute-selector 'class "mw-spcontent") page))) + (define next ((query-selector (attribute-selector 'class "mw-nextlink") page-content))) + (define next-offset + (if next + (second (regexp-match #rx"offset=([0-9]*)" (get-attribute 'href (bits->attributes next)))) + #f)) + (define list-item-generator (query-selector (λ (e a c) (eq? e 'li)) page-content)) + (define these-items (convert-list-items list-item-generator)) + (define all-items (append items these-items)) + (printf "page offset \"~a\" has ~a items (~a so far)~n" offset (length these-items) (length all-items)) + (if next + (get-items-recursive next-offset all-items) + all-items)) + +(call-with-output-file output-file #:exists 'truncate/replace + (λ (out) + (write-json (get-items-recursive) out)))