Fix running out of file descriptors

This commit is contained in:
Cadence Ember 2024-05-01 00:53:09 +12:00
parent a52d131b93
commit 6260ba809b
12 changed files with 134 additions and 100 deletions

View file

@ -1,6 +1,5 @@
#lang racket/base #lang racket/base
(require (prefix-in easy: net/http-easy) (require "../src/data.rkt"
"../src/data.rkt"
"xexpr-utils.rkt") "xexpr-utils.rkt")
(provide (provide

View file

@ -22,8 +22,6 @@
(provide (provide
; headers to always send on all http responses ; headers to always send on all http responses
always-headers always-headers
; timeout durations for http-easy requests
timeouts
; generates a consistent footer ; generates a consistent footer
application-footer application-footer
; generates a consistent template for wiki page content to sit in ; generates a consistent template for wiki page content to sit in
@ -39,7 +37,6 @@
(define always-headers (define always-headers
(list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom (list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom
(header #"Link" (string->bytes/latin-1 link-header)))) (header #"Link" (string->bytes/latin-1 link-header))))
(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
(define-runtime-path path-static "../static") (define-runtime-path path-static "../static")
(define theme-icons (define theme-icons

View file

@ -7,8 +7,8 @@
(prefix-in easy: net/http-easy) (prefix-in easy: net/http-easy)
db db
memo memo
"fandom-request.rkt"
"static-data.rkt" "static-data.rkt"
"whole-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt" "../lib/xexpr-utils.rkt"
"../archiver/archiver-database.rkt" "../archiver/archiver-database.rkt"
@ -54,16 +54,14 @@
(vector-ref row 3))) (vector-ref row 3)))
siteinfo-default)] siteinfo-default)]
[else [else
(define dest-url (define res
(format "https://~a.fandom.com/api.php?~a" (fandom-get-api
wikiname wikiname
(params->query '(("action" . "query") (params->query '(("action" . "query")
("meta" . "siteinfo") ("meta" . "siteinfo")
("siprop" . "general|rightsinfo") ("siprop" . "general|rightsinfo")
("format" . "json") ("format" . "json")
("formatversion" . "2"))))) ("formatversion" . "2")))))
(log-outgoing dest-url)
(define res (easy:get dest-url))
(define data (easy:response-json res)) (define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data) (siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))

View file

@ -33,12 +33,29 @@
; don't forget that I'm returning *code* - return a call to the function ; don't forget that I'm returning *code* - return a call to the function
(datum->syntax stx `(make-dispatcher-tree ,ds))) (datum->syntax stx `(make-dispatcher-tree ,ds)))
; guard that the page returned a response, otherwise print more detailed debugging information
(define-syntax-rule (page ds name)
(λ (req)
(define dispatcher (hash-ref ds (quote name)))
(define page-response (dispatcher req))
(if (response? page-response)
page-response
(response/output
#:code 500
#:mime-type #"text/plain"
(λ (out)
(for ([port (list (current-error-port) out)])
(parameterize ([current-output-port port])
(printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n"
(quote name)
page-response))))))))
(define (make-dispatcher-tree ds) (define (make-dispatcher-tree ds)
(define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher)) (define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher))
(define tree (define tree
(sequencer:make (sequencer:make
subdomain-dispatcher subdomain-dispatcher
(pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/" (page ds page-home))
(pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/proxy" (hash-ref ds 'page-proxy))
(pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (pathprocedure:make "/search" (hash-ref ds 'page-global-search))
(pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) (pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings))
@ -48,7 +65,7 @@
(if (config-true? 'feature_offline::enabled) (if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline)))
(λ (_conn _req) (next-dispatcher))) (λ (_conn _req) (next-dispatcher)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home)))
(if (config-true? 'feature_offline::enabled) (if (config-true? 'feature_offline::enabled)

48
src/fandom-request.rkt Normal file
View file

@ -0,0 +1,48 @@
#lang typed/racket/base
(require "config.rkt"
"../lib/url-utils.rkt")
(define-type Headers (HashTable Symbol (U Bytes String)))
(require/typed net/http-easy
[#:opaque Timeout-Config timeout-config?]
[#:opaque Response response?]
[#:opaque Session session?]
[current-session (Parameter Session)]
[make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)]
[get ((U Bytes String)
[#:close? Boolean]
[#:headers Headers]
[#:timeouts Timeout-Config]
[#:max-attempts Exact-Positive-Integer]
[#:max-redirects Exact-Nonnegative-Integer]
[#:user-agent (U Bytes String)]
-> Response)])
(provide
fandom-get
fandom-get-api
timeouts)
(define timeouts (make-timeout-config #:lease 5 #:connect 5))
(: no-headers Headers)
(define no-headers '#hasheq())
(: fandom-get (String String [#:headers (Option Headers)] -> Response))
(define (fandom-get wikiname path #:headers [headers #f])
(define dest-url (string-append "https://www.fandom.com" path))
(define host (string-append wikiname ".fandom.com"))
(log-outgoing wikiname path)
(get dest-url
#:timeouts timeouts
#:headers (hash-set (or headers no-headers) 'Host host)))
(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response))
(define (fandom-get-api wikiname params #:headers [headers #f])
(fandom-get wikiname
(string-append "/api.php?" (params->query params))
#:headers headers))
(: log-outgoing (String String -> Void))
(define (log-outgoing wikiname path)
(when (config-true? 'log_outgoing)
(printf "out: ~a ~a~n" wikiname path)))

View file

@ -15,11 +15,11 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"page-wiki.rkt" "page-wiki.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -73,30 +73,24 @@
(define-values (members-data page-data siteinfo) (define-values (members-data page-data siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (easy:response-json
(format "~a/api.php?~a" (fandom-get-api
origin wikiname
(params->query `(("action" . "query") `(("action" . "query")
("list" . "categorymembers") ("list" . "categorymembers")
("cmtitle" . ,prefixed-category) ("cmtitle" . ,prefixed-category)
("cmlimit" . "max") ("cmlimit" . "max")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ () (λ ()
(define dest-url (easy:response-json
(format "~a/api.php?~a" (fandom-get-api
origin wikiname
(params->query `(("action" . "parse") `(("action" . "parse")
("page" . ,prefixed-category) ("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks") ("prop" . "text|headhtml|langlinks")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))

View file

@ -15,11 +15,11 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"page-wiki.rkt" "page-wiki.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide page-file) (provide page-file)
@ -40,8 +40,7 @@
(imageDescription . #f)))) (imageDescription . #f))))
(define (url-content-type url) (define (url-content-type url)
(log-outgoing url) (define dest-res (easy:head url))
(define dest-res (easy:head url #:timeouts timeouts))
(easy:response-headers-ref dest-res 'content-type)) (easy:response-headers-ref dest-res 'content-type))
(define (get-media-html url content-type) (define (get-media-html url content-type)
@ -106,20 +105,18 @@
(response-handler (response-handler
(define wikiname (path/param-path (first (url-path (request-uri req))))) (define wikiname (path/param-path (first (url-path (request-uri req)))))
(define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) (define prefixed-title (path/param-path (caddr (url-path (request-uri req)))))
(define origin (format "https://~a.fandom.com" wikiname)) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title))
(define source-url (format "~a/wiki/~a" origin prefixed-title))
(define-values (media-detail siteinfo) (define-values (media-detail siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (define dest-res
(format "~a/wikia.php?~a" (fandom-get
origin wikiname
(format "/wikia.php?~a"
(params->query `(("format" . "json") ("controller" . "Lightbox") (params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail") ("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title))))) ("fileTitle" . ,prefixed-title))))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res)) (easy:response-json dest-res))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))

View file

@ -2,7 +2,6 @@
(require racket/dict (require racket/dict
racket/list racket/list
racket/string racket/string
(prefix-in easy: net/http-easy)
; html libs ; html libs
html-writing html-writing
; web server libs ; web server libs
@ -18,7 +17,6 @@
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide

View file

@ -17,12 +17,12 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"../lib/pure-utils.rkt" "../lib/pure-utils.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/tree-updater.rkt" "../lib/tree-updater.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -38,24 +38,19 @@
(define (page-wiki req) (define (page-wiki req)
(define wikiname (path/param-path (first (url-path (request-uri req))))) (define wikiname (path/param-path (first (url-path (request-uri req)))))
(define user-cookies (user-cookies-getter req)) (define user-cookies (user-cookies-getter req))
(define origin (format "https://~a.fandom.com" wikiname))
(define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define-values (dest-res siteinfo) (define-values (dest-res siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (fandom-get-api
(format "~a/api.php?~a" wikiname
origin `(("action" . "parse")
(params->query `(("action" . "parse")
("page" . ,path) ("page" . ,path)
("prop" . "text|headhtml|langlinks") ("prop" . "text|headhtml|langlinks")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json"))
(log-outgoing dest-url)
(easy:get dest-url
#:timeouts timeouts
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))
@ -103,4 +98,13 @@
#:code 200 #:code 200
#:headers headers #:headers headers
(λ (out) (λ (out)
(write-html body out))))))])) (write-html body out))))))]
[(eq? 404 (easy:response-status-code dest-res))
(next-dispatcher)]
[else
(response-handler
(error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
wikiname
path
(easy:response-status-code dest-res)
(easy:response-body dest-res)))]))

View file

@ -3,8 +3,8 @@
(prefix-in easy: net/http-easy) (prefix-in easy: net/http-easy)
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"fandom-request.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -17,20 +17,14 @@
'(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))) '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
(define (search-fandom wikiname query params) (define (search-fandom wikiname query params)
;; constructing the URL where I want to get fandom data from... (define res
(define origin (format "https://~a.fandom.com" wikiname)) (fandom-get-api
;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json wikiname
(define dest-url `(("action" . "query")
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "search") ("list" . "search")
("srsearch" . ,query) ("srsearch" . ,query)
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json"))))
;; HTTP request to dest-url for search results
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res)) (define json (easy:response-json res))
(define search-results (jp "/query/search" json)) (define search-results (jp "/query/search" json))
(generate-results-content-fandom wikiname query search-results)) (generate-results-content-fandom wikiname query search-results))

View file

@ -5,7 +5,6 @@
"application-globals.rkt" "application-globals.rkt"
"../lib/html-parsing/main.rkt" "../lib/html-parsing/main.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -37,7 +36,7 @@
("sort" . ,(cdr sort)))))) ("sort" . ,(cdr sort))))))
;; HTTP request to dest-url for search results ;; HTTP request to dest-url for search results
(log-outgoing dest-url) (log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts)) (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5)))
(define json (easy:response-json res)) (define json (easy:response-json res))
;; build result objects ;; build result objects

View file

@ -1,11 +0,0 @@
#lang typed/racket/base
(require "config.rkt")
(provide
; prints "out: <url>"
log-outgoing)
(: log-outgoing (String -> Void))
(define (log-outgoing url-string)
(when (config-true? 'log_outgoing)
(printf "out: ~a~n" url-string)))