cadence
/
breezewiki
3
4
Fork 4
Code Issues Pull Requests 3 Releases Wiki Activity

Compare commits

base: cadence:main
Branches Tags
cadence:main
cadence:disable-requests
cadence:early-hints
Evalprime:tardis
Evalprime:main
Evalprime:disable-requests
Evalprime:early-hints
..
compare: Evalprime:main
Branches Tags
Evalprime:tardis
Evalprime:main
Evalprime:disable-requests
Evalprime:early-hints
cadence:main
cadence:disable-requests
cadence:early-hints

No commits in common. "main" and "main" have entirely different histories.
main ... main

12 changed files with 116 additions and 151 deletions
Show Stats Expand all files Collapse all files

3
lib/thread-utils.rkt
View File

@ -1,5 +1,6 @@
#lang racket/base
(require "../src/data.rkt"
(require (prefix-in easy: net/http-easy)
"../src/data.rkt"
"xexpr-utils.rkt")
(provide

3
src/application-globals.rkt
View File

@ -22,6 +22,8 @@
(provide
; headers to always send on all http responses
always-headers
; timeout durations for http-easy requests
timeouts
; generates a consistent footer
application-footer
; generates a consistent template for wiki page content to sit in
@ -37,6 +39,7 @@
(define always-headers
(list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom
(header #"Link" (string->bytes/latin-1 link-header))))
(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
(define-runtime-path path-static "../static")
(define theme-icons

32
src/data.rkt
View File

@ -7,8 +7,8 @@
(prefix-in easy: net/http-easy)
db
memo
"fandom-request.rkt"
"static-data.rkt"
"whole-utils.rkt"
"../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt"
"../archiver/archiver-database.rkt"
@ -54,21 +54,21 @@
(vector-ref row 3)))
siteinfo-default)]
[else
(define res
(fandom-get-api
wikiname
'(("action" . "query")
("meta" . "siteinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2"))))
(cond [(= (easy:response-status-code res) 200)
(define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))]
[else siteinfo-default])]))
(define dest-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query '(("action" . "query")
("meta" . "siteinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2")))))
(log-outgoing dest-url)
(define res (easy:get dest-url))
(define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))]))
(define/memoize (head-data-getter wikiname) #:hash hash
;; data will be stored here, can be referenced by the memoized closure

41
src/dispatcher-tree.rkt
View File

@ -33,43 +33,26 @@
; don't forget that I'm returning *code* - return a call to the function
(datum->syntax stx `(make-dispatcher-tree ,ds)))
; guard that the page returned a response, otherwise print more detailed debugging information
(define-syntax-rule (page ds name)
(λ (req)
(define dispatcher (hash-ref ds (quote name)))
(define page-response (dispatcher req))
(if (response? page-response)
page-response
(response/output
#:code 500
#:mime-type #"text/plain"
(λ (out)
(for ([port (list (current-error-port) out)])
(parameterize ([current-output-port port])
(printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n"
(quote name)
page-response))))))))
(define (make-dispatcher-tree ds)
(define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher))
(define tree
(sequencer:make
subdomain-dispatcher
(pathprocedure:make "/" (page ds page-home))
(pathprocedure:make "/proxy" (page ds page-proxy))
(pathprocedure:make "/search" (page ds page-global-search))
(pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
(pathprocedure:make "/" (hash-ref ds 'page-home))
(pathprocedure:make "/proxy" (hash-ref ds 'page-proxy))
(pathprocedure:make "/search" (hash-ref ds 'page-global-search))
(pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file)))
(if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline)))
(λ (_conn _req) (next-dispatcher)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home)))
(if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive)))
(filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive)))
(λ (_conn _req) (next-dispatcher)))
(hash-ref ds 'static-dispatcher)
(lift:make (hash-ref ds 'page-not-found))))

48
src/fandom-request.rkt
View File

@ -1,48 +0,0 @@
#lang typed/racket/base
(require "config.rkt"
"../lib/url-utils.rkt")
(define-type Headers (HashTable Symbol (U Bytes String)))
(require/typed net/http-easy
[#:opaque Timeout-Config timeout-config?]
[#:opaque Response response?]
[#:opaque Session session?]
[current-session (Parameter Session)]
[make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)]
[get ((U Bytes String)
[#:close? Boolean]
[#:headers Headers]
[#:timeouts Timeout-Config]
[#:max-attempts Exact-Positive-Integer]
[#:max-redirects Exact-Nonnegative-Integer]
[#:user-agent (U Bytes String)]
-> Response)])
(provide
fandom-get
fandom-get-api
timeouts)
(define timeouts (make-timeout-config #:lease 5 #:connect 5))
(: no-headers Headers)
(define no-headers '#hasheq())
(: fandom-get (String String [#:headers (Option Headers)] -> Response))
(define (fandom-get wikiname path #:headers [headers #f])
(define dest-url (string-append "https://www.fandom.com" path))
(define host (string-append wikiname ".fandom.com"))
(log-outgoing wikiname path)
(get dest-url
#:timeouts timeouts
#:headers (hash-set (or headers no-headers) 'Host host)))
(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response))
(define (fandom-get-api wikiname params #:headers [headers #f])
(fandom-get wikiname
(string-append "/api.php?" (params->query params))
#:headers headers))
(: log-outgoing (String String -> Void))
(define (log-outgoing wikiname path)
(when (config-true? 'log_outgoing)
(printf "out: ~a ~a~n" wikiname path)))

42
src/page-category.rkt
View File

@ -15,11 +15,11 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"page-wiki.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -73,24 +73,30 @@
(define-values (members-data page-data siteinfo)
(thread-values
(λ ()
(easy:response-json
(fandom-get-api
wikiname
`(("action" . "query")
("list" . "categorymembers")
("cmtitle" . ,prefixed-category)
("cmlimit" . "max")
("formatversion" . "2")
("format" . "json")))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "categorymembers")
("cmtitle" . ,prefixed-category)
("cmlimit" . "max")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(easy:response-json
(fandom-get-api
wikiname
`(("action" . "parse")
("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "parse")
("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(siteinfo-fetch wikiname))))

23
src/page-file.rkt
View File

@ -15,11 +15,11 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"page-wiki.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide page-file)
@ -40,7 +40,8 @@
(imageDescription . #f))))
(define (url-content-type url)
(define dest-res (easy:head url))
(log-outgoing url)
(define dest-res (easy:head url #:timeouts timeouts))
(easy:response-headers-ref dest-res 'content-type))
(define (get-media-html url content-type)
@ -105,18 +106,20 @@
(response-handler
(define wikiname (path/param-path (first (url-path (request-uri req)))))
(define prefixed-title (path/param-path (caddr (url-path (request-uri req)))))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title))
(define origin (format "https://~a.fandom.com" wikiname))
(define source-url (format "~a/wiki/~a" origin prefixed-title))
(define-values (media-detail siteinfo)
(thread-values
(λ ()
(define dest-res
(fandom-get
wikiname
(format "/wikia.php?~a"
(params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title))))))
(define dest-url
(format "~a/wikia.php?~a"
origin
(params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title)))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(siteinfo-fetch wikiname))))

2
src/page-search.rkt
View File

@ -2,6 +2,7 @@
(require racket/dict
racket/list
racket/string
(prefix-in easy: net/http-easy)
; html libs
html-writing
; web server libs
@ -17,6 +18,7 @@
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide

34
src/page-wiki.rkt
View File

@ -17,12 +17,12 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"../lib/pure-utils.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/tree-updater.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -38,20 +38,25 @@
(define (page-wiki req)
(define wikiname (path/param-path (first (url-path (request-uri req)))))
(define user-cookies (user-cookies-getter req))
(define origin (format "https://~a.fandom.com" wikiname))
(define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define-values (dest-res siteinfo)
(thread-values
(λ ()
(fandom-get-api
wikiname
`(("action" . "parse")
("page" . ,path)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json"))
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "parse")
("page" . ,path)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(easy:get dest-url
#:timeouts timeouts
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(λ ()
(siteinfo-fetch wikiname))))
@ -98,13 +103,4 @@
#:code 200
#:headers headers
(λ (out)
(write-html body out))))))]
[(eq? 404 (easy:response-status-code dest-res))
(next-dispatcher)]
[else
(response-handler
(error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
wikiname
path
(easy:response-status-code dest-res)
(easy:response-body dest-res)))]))
(write-html body out))))))]))

24
src/search-provider-fandom.rkt
View File

@ -3,8 +3,8 @@
(prefix-in easy: net/http-easy)
"application-globals.rkt"
"config.rkt"
"fandom-request.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -17,14 +17,20 @@
'(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
(define (search-fandom wikiname query params)
(define res
(fandom-get-api
wikiname
`(("action" . "query")
("list" . "search")
("srsearch" . ,query)
("formatversion" . "2")
("format" . "json"))))
;; constructing the URL where I want to get fandom data from...
(define origin (format "https://~a.fandom.com" wikiname))
;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "search")
("srsearch" . ,query)
("formatversion" . "2")
("format" . "json")))))
;; HTTP request to dest-url for search results
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res))
(define search-results (jp "/query/search" json))
(generate-results-content-fandom wikiname query search-results))

4
src/search-provider-solr.rkt
View File

@ -5,6 +5,7 @@
"application-globals.rkt"
"../lib/html-parsing/main.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -35,7 +36,8 @@
("hl.tag.post" . "</mark>")
("sort" . ,(cdr sort))))))
;; HTTP request to dest-url for search results
(define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5)))
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res))
;; build result objects

11
src/whole-utils.rkt Normal file
View File

@ -0,0 +1,11 @@
#lang typed/racket/base
(require "config.rkt")
(provide
; prints "out: <url>"
log-outgoing)
(: log-outgoing (String -> Void))
(define (log-outgoing url-string)
(when (config-true? 'log_outgoing)
(printf "out: ~a~n" url-string)))