Compare commits

..

15 commits

Author SHA1 Message Date
9386d1c93e
Hide slideshows 2024-06-05 21:13:26 +10:00
755efe3cd6 Tabber code size and quality 2024-06-05 23:07:05 +12:00
47d92d3a37 Merge PR #15 2024-06-05 21:53:03 +12:00
2e0bd786ec add tardis 2024-06-05 21:50:09 +12:00
0fd0efc3f2 Use default siteinfo when online wiki not found 2024-05-04 18:01:50 +12:00
d2765c2a78 Fix duplicate params->query 2024-05-02 00:01:32 +12:00
7dff049ece Wrap all pages in response safety checker 2024-05-01 00:57:13 +12:00
6260ba809b Fix running out of file descriptors 2024-05-01 00:53:09 +12:00
a52d131b93 Split massive uploads in Solr indexer 2024-01-11 22:36:16 +13:00
b02e2a4053 Fix failing test after search was refactored 2024-01-10 11:21:42 +13:00
2b3a8fe108
Fix scrolling to sections if a tab's hash coincides with one
ben10/wiki/Alien_X_(Classic)#Appearances
2023-11-13 14:35:35 +11:00
dcb8a8a590
Prevent making duplicate history entries 2023-11-06 20:31:20 +11:00
f5399524b1
Prevent linking to tabs with no IDs 2023-11-06 20:15:18 +11:00
ead6896818
Add the ability to specify/open the last open tab in the URL 2023-11-06 20:15:18 +11:00
9773e62c46
Add better support for tabs
Some pages break without actual tab support, such as
https://breezewiki.com/ben10/wiki/Ultimatrix_(Original)#Modes

This change aims to work with old browsers (such as Firefox for Android 68)
and browsers with Javascript disabled (by showing all tab contents and hiding
the tab bar, i.e. how tabs work before this change).
2023-11-06 20:15:15 +11:00
15 changed files with 238 additions and 206 deletions

View file

@ -8,6 +8,7 @@
racket/promise racket/promise
racket/port racket/port
racket/runtime-path racket/runtime-path
racket/sequence
racket/string racket/string
file/gunzip file/gunzip
db db
@ -156,8 +157,16 @@
(define data (define data
(cond (cond
[(and (read-from-cache?) (file-exists? "cache.rkt")) [(and (read-from-cache?) (file-exists? "cache.rkt"))
(displayln "Reading in...") (define size (file-size "cache.rkt"))
(with-input-from-file "cache.rkt" (λ () (read)))] (call-with-input-file "cache.rkt"
(λ (in)
(define quit (make-progress (λ () (progress^ (ceiling (/ (file-position in) 64 1024))
(ceiling (/ size 64 1024))
"Reading in..."))
2))
(begin0
(read in)
(quit))))]
[else [else
(define x (box (progress^ 0 1 "..."))) (define x (box (progress^ 0 1 "...")))
(define quit (make-progress (λ () (unbox x)))) (define quit (make-progress (λ () (unbox x))))
@ -183,11 +192,15 @@
(display "Converting... ") (display "Converting... ")
(flush-output) (flush-output)
(define ser (jsexpr->bytes data)) (define slice-size 30000)
(define slices (ceiling (/ (length data) slice-size)))
(for ([slice (in-slice slice-size data)]
[i (in-naturals 1)])
(define ser (jsexpr->bytes slice))
(define ser-port (open-input-bytes ser)) (define ser-port (open-input-bytes ser))
(define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024))
(ceiling (/ (bytes-length ser) 64 1024)) (ceiling (/ (bytes-length ser) 64 1024))
"Posting...")) (format "Posting... (~a/~a)" i slices)))
2)) 2))
(define res (define res
(post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname)
@ -195,6 +208,6 @@
#:headers '#hasheq((Content-Type . "application/json")) #:headers '#hasheq((Content-Type . "application/json"))
#:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300)))
(quit) (quit)
(displayln (response-status-line res))) (displayln (response-status-line res))))
(run start) (run start)

View file

@ -1,6 +1,5 @@
#lang racket/base #lang racket/base
(require (prefix-in easy: net/http-easy) (require "../src/data.rkt"
"../src/data.rkt"
"xexpr-utils.rkt") "xexpr-utils.rkt")
(provide (provide

View file

@ -22,8 +22,6 @@
(provide (provide
; headers to always send on all http responses ; headers to always send on all http responses
always-headers always-headers
; timeout durations for http-easy requests
timeouts
; generates a consistent footer ; generates a consistent footer
application-footer application-footer
; generates a consistent template for wiki page content to sit in ; generates a consistent template for wiki page content to sit in
@ -39,7 +37,6 @@
(define always-headers (define always-headers
(list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom (list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom
(header #"Link" (string->bytes/latin-1 link-header)))) (header #"Link" (string->bytes/latin-1 link-header))))
(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
(define-runtime-path path-static "../static") (define-runtime-path path-static "../static")
(define theme-icons (define theme-icons
@ -113,7 +110,7 @@
(div (@ (class "niwa__left")) (div (@ (class "niwa__left"))
(p ,((extwiki-group^-description group) props)) (p ,((extwiki-group^-description group) props))
(p ,((extwiki^-description xt) props)) (p ,((extwiki^-description xt) props))
(p "This wiki's core community has wholly migrated away from Fandom. You should " (p "This wiki's core community has largely migrated away from Fandom. You should "
(a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!")) (a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!"))
(p (@ (class "niwa__feedback")) (p (@ (class "niwa__feedback"))
,@(add-between ,@(add-between
@ -175,7 +172,7 @@
(define styles (define styles
(list (list
(format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies)) (format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies))
(format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin))) (format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.fandom.photoGallery.gallery.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin)))
(if (config-true? 'strict_proxy) (if (config-true? 'strict_proxy)
(map u-proxy-url styles) (map u-proxy-url styles)
styles)] styles)]

View file

@ -7,8 +7,8 @@
(prefix-in easy: net/http-easy) (prefix-in easy: net/http-easy)
db db
memo memo
"fandom-request.rkt"
"static-data.rkt" "static-data.rkt"
"whole-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt" "../lib/xexpr-utils.rkt"
"../archiver/archiver-database.rkt" "../archiver/archiver-database.rkt"
@ -54,21 +54,21 @@
(vector-ref row 3))) (vector-ref row 3)))
siteinfo-default)] siteinfo-default)]
[else [else
(define dest-url (define res
(format "https://~a.fandom.com/api.php?~a" (fandom-get-api
wikiname wikiname
(params->query '(("action" . "query") '(("action" . "query")
("meta" . "siteinfo") ("meta" . "siteinfo")
("siprop" . "general|rightsinfo") ("siprop" . "general|rightsinfo")
("format" . "json") ("format" . "json")
("formatversion" . "2"))))) ("formatversion" . "2"))))
(log-outgoing dest-url) (cond [(= (easy:response-status-code res) 200)
(define res (easy:get dest-url))
(define data (easy:response-json res)) (define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data) (siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data))) (second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data) (license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))])) (jp "/query/rightsinfo/url" data)))]
[else siteinfo-default])]))
(define/memoize (head-data-getter wikiname) #:hash hash (define/memoize (head-data-getter wikiname) #:hash hash
;; data will be stored here, can be referenced by the memoized closure ;; data will be stored here, can be referenced by the memoized closure

View file

@ -33,26 +33,43 @@
; don't forget that I'm returning *code* - return a call to the function ; don't forget that I'm returning *code* - return a call to the function
(datum->syntax stx `(make-dispatcher-tree ,ds))) (datum->syntax stx `(make-dispatcher-tree ,ds)))
; guard that the page returned a response, otherwise print more detailed debugging information
(define-syntax-rule (page ds name)
(λ (req)
(define dispatcher (hash-ref ds (quote name)))
(define page-response (dispatcher req))
(if (response? page-response)
page-response
(response/output
#:code 500
#:mime-type #"text/plain"
(λ (out)
(for ([port (list (current-error-port) out)])
(parameterize ([current-output-port port])
(printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n"
(quote name)
page-response))))))))
(define (make-dispatcher-tree ds) (define (make-dispatcher-tree ds)
(define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher)) (define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher))
(define tree (define tree
(sequencer:make (sequencer:make
subdomain-dispatcher subdomain-dispatcher
(pathprocedure:make "/" (hash-ref ds 'page-home)) (pathprocedure:make "/" (page ds page-home))
(pathprocedure:make "/proxy" (hash-ref ds 'page-proxy)) (pathprocedure:make "/proxy" (page ds page-proxy))
(pathprocedure:make "/search" (hash-ref ds 'page-global-search)) (pathprocedure:make "/search" (page ds page-global-search))
(pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings)) (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works)) (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category))) (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file))) (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
(if (config-true? 'feature_offline::enabled) (if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
(λ (_conn _req) (next-dispatcher))) (λ (_conn _req) (next-dispatcher)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki))) (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search))) (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home))) (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))
(if (config-true? 'feature_offline::enabled) (if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive))) (filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive)))
(λ (_conn _req) (next-dispatcher))) (λ (_conn _req) (next-dispatcher)))
(hash-ref ds 'static-dispatcher) (hash-ref ds 'static-dispatcher)
(lift:make (hash-ref ds 'page-not-found)))) (lift:make (hash-ref ds 'page-not-found))))

View file

@ -71,6 +71,13 @@
(λ (props) (λ (props)
'(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop."))) '(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop.")))
'Tardis
(extwiki-group^
"Tardis"
'(("Forking announcement" . "https://tardis.wiki/wiki/Tardis:Forking_announcement")
("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/"))
(λ (props) '()))
'empty 'empty
(extwiki-group^ (extwiki-group^
"Misc" "Misc"
@ -419,6 +426,15 @@
(λ (props) (λ (props)
`())) `()))
(extwiki^
'("tardis") 'default
'Tardis
"Tardis Data Core"
"https://tardis.wiki/wiki/Doctor_Who_Wiki"
"https://tardis.wiki/images/Tardis_images/e/e6/Site-logo.png"
(λ (props)
`()))
;; fandom wikinames * empty * empty * Name * Home Page ;; fandom wikinames * empty * empty * Name * Home Page
(extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f) (extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f)
(extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f) (extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f)

48
src/fandom-request.rkt Normal file
View file

@ -0,0 +1,48 @@
#lang typed/racket/base
(require "config.rkt"
"../lib/url-utils.rkt")
(define-type Headers (HashTable Symbol (U Bytes String)))
(require/typed net/http-easy
[#:opaque Timeout-Config timeout-config?]
[#:opaque Response response?]
[#:opaque Session session?]
[current-session (Parameter Session)]
[make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)]
[get ((U Bytes String)
[#:close? Boolean]
[#:headers Headers]
[#:timeouts Timeout-Config]
[#:max-attempts Exact-Positive-Integer]
[#:max-redirects Exact-Nonnegative-Integer]
[#:user-agent (U Bytes String)]
-> Response)])
(provide
fandom-get
fandom-get-api
timeouts)
(define timeouts (make-timeout-config #:lease 5 #:connect 5))
(: no-headers Headers)
(define no-headers '#hasheq())
(: fandom-get (String String [#:headers (Option Headers)] -> Response))
(define (fandom-get wikiname path #:headers [headers #f])
(define dest-url (string-append "https://www.fandom.com" path))
(define host (string-append wikiname ".fandom.com"))
(log-outgoing wikiname path)
(get dest-url
#:timeouts timeouts
#:headers (hash-set (or headers no-headers) 'Host host)))
(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response))
(define (fandom-get-api wikiname params #:headers [headers #f])
(fandom-get wikiname
(string-append "/api.php?" (params->query params))
#:headers headers))
(: log-outgoing (String String -> Void))
(define (log-outgoing wikiname path)
(when (config-true? 'log_outgoing)
(printf "out: ~a ~a~n" wikiname path)))

View file

@ -15,11 +15,11 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"page-wiki.rkt" "page-wiki.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -73,30 +73,24 @@
(define-values (members-data page-data siteinfo) (define-values (members-data page-data siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (easy:response-json
(format "~a/api.php?~a" (fandom-get-api
origin wikiname
(params->query `(("action" . "query") `(("action" . "query")
("list" . "categorymembers") ("list" . "categorymembers")
("cmtitle" . ,prefixed-category) ("cmtitle" . ,prefixed-category)
("cmlimit" . "max") ("cmlimit" . "max")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ () (λ ()
(define dest-url (easy:response-json
(format "~a/api.php?~a" (fandom-get-api
origin wikiname
(params->query `(("action" . "parse") `(("action" . "parse")
("page" . ,prefixed-category) ("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks") ("prop" . "text|headhtml|langlinks")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))

View file

@ -15,11 +15,11 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"page-wiki.rkt" "page-wiki.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide page-file) (provide page-file)
@ -40,8 +40,7 @@
(imageDescription . #f)))) (imageDescription . #f))))
(define (url-content-type url) (define (url-content-type url)
(log-outgoing url) (define dest-res (easy:head url))
(define dest-res (easy:head url #:timeouts timeouts))
(easy:response-headers-ref dest-res 'content-type)) (easy:response-headers-ref dest-res 'content-type))
(define (get-media-html url content-type) (define (get-media-html url content-type)
@ -106,20 +105,18 @@
(response-handler (response-handler
(define wikiname (path/param-path (first (url-path (request-uri req))))) (define wikiname (path/param-path (first (url-path (request-uri req)))))
(define prefixed-title (path/param-path (caddr (url-path (request-uri req))))) (define prefixed-title (path/param-path (caddr (url-path (request-uri req)))))
(define origin (format "https://~a.fandom.com" wikiname)) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title))
(define source-url (format "~a/wiki/~a" origin prefixed-title))
(define-values (media-detail siteinfo) (define-values (media-detail siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (define dest-res
(format "~a/wikia.php?~a" (fandom-get
origin wikiname
(format "/wikia.php?~a"
(params->query `(("format" . "json") ("controller" . "Lightbox") (params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail") ("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title))))) ("fileTitle" . ,prefixed-title))))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res)) (easy:response-json dest-res))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))

View file

@ -2,7 +2,6 @@
(require racket/dict (require racket/dict
racket/list racket/list
racket/string racket/string
(prefix-in easy: net/http-easy)
; html libs ; html libs
html-writing html-writing
; web server libs ; web server libs
@ -18,15 +17,14 @@
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
page-search) page-search)
(define search-providers (define search-providers
(hash "fandom" generate-results-content-fandom (hash "fandom" search-fandom
"solr" generate-results-content-solr)) "solr" search-solr))
;; this takes the info we gathered from fandom and makes the big fat x-expression page ;; this takes the info we gathered from fandom and makes the big fat x-expression page
(define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f]) (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f])

View file

@ -17,12 +17,12 @@
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"data.rkt" "data.rkt"
"fandom-request.rkt"
"../lib/pure-utils.rkt" "../lib/pure-utils.rkt"
"../lib/syntax.rkt" "../lib/syntax.rkt"
"../lib/thread-utils.rkt" "../lib/thread-utils.rkt"
"../lib/tree-updater.rkt" "../lib/tree-updater.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
@ -38,24 +38,19 @@
(define (page-wiki req) (define (page-wiki req)
(define wikiname (path/param-path (first (url-path (request-uri req))))) (define wikiname (path/param-path (first (url-path (request-uri req)))))
(define user-cookies (user-cookies-getter req)) (define user-cookies (user-cookies-getter req))
(define origin (format "https://~a.fandom.com" wikiname))
(define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/")) (define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)) (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define-values (dest-res siteinfo) (define-values (dest-res siteinfo)
(thread-values (thread-values
(λ () (λ ()
(define dest-url (fandom-get-api
(format "~a/api.php?~a" wikiname
origin `(("action" . "parse")
(params->query `(("action" . "parse")
("page" . ,path) ("page" . ,path)
("prop" . "text|headhtml|langlinks") ("prop" . "text|headhtml|langlinks")
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json"))
(log-outgoing dest-url)
(easy:get dest-url
#:timeouts timeouts
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))) #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(λ () (λ ()
(siteinfo-fetch wikiname)))) (siteinfo-fetch wikiname))))
@ -103,4 +98,13 @@
#:code 200 #:code 200
#:headers headers #:headers headers
(λ (out) (λ (out)
(write-html body out))))))])) (write-html body out))))))]
[(eq? 404 (easy:response-status-code dest-res))
(next-dispatcher)]
[else
(response-handler
(error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
wikiname
path
(easy:response-status-code dest-res)
(easy:response-body dest-res)))]))

View file

@ -3,38 +3,34 @@
(prefix-in easy: net/http-easy) (prefix-in easy: net/http-easy)
"application-globals.rkt" "application-globals.rkt"
"config.rkt" "config.rkt"
"fandom-request.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
generate-results-content-fandom) search-fandom)
(module+ test (module+ test
(require rackunit (require rackunit
"test-utils.rkt") "test-utils.rkt")
(define search-json-data (define search-results-data
'#hasheq((batchcomplete . #t) (query . #hasheq((search . (#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181))))))))) '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
(define (generate-results-content-fandom wikiname query params) (define (search-fandom wikiname query params)
;; constructing the URL where I want to get fandom data from... (define res
(define origin (format "https://~a.fandom.com" wikiname)) (fandom-get-api
;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json wikiname
(define dest-url `(("action" . "query")
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "search") ("list" . "search")
("srsearch" . ,query) ("srsearch" . ,query)
("formatversion" . "2") ("formatversion" . "2")
("format" . "json"))))) ("format" . "json"))))
;; HTTP request to dest-url for search results
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res)) (define json (easy:response-json res))
(define search-results (jp "/query/search" json)) (define search-results (jp "/query/search" json))
(generate-results-content-fandom wikiname query search-results))
;; generate content for display in the wiki page layout ;;; generate content for display in the wiki page layout
(define (generate-results-content-fandom wikiname query search-results)
`(div (@ (class "mw-parser-output")) `(div (@ (class "mw-parser-output"))
;; header before the search results showing how many we found ;; header before the search results showing how many we found
(p ,(format "~a results found for " (length search-results)) (p ,(format "~a results found for " (length search-results))
@ -60,4 +56,4 @@
(module+ test (module+ test
(parameterize ([(config-parameter 'feature_offline::only) "false"]) (parameterize ([(config-parameter 'feature_offline::only) "false"])
(check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule") (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")
(generate-results-content-fandom test-req "" "test" "Gacha" search-json-data)))))) (generate-results-content-fandom "test" "Gacha" search-results-data))))))

View file

@ -5,15 +5,14 @@
"application-globals.rkt" "application-globals.rkt"
"../lib/html-parsing/main.rkt" "../lib/html-parsing/main.rkt"
"../lib/url-utils.rkt" "../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt") "../lib/xexpr-utils.rkt")
(provide (provide
generate-results-content-solr) search-solr)
(struct result^ (hl-title hl-body kb words page-path) #:transparent) (struct result^ (hl-title hl-body kb words page-path) #:transparent)
(define (generate-results-content-solr wikiname query params) (define (search-solr wikiname query params)
;; grab things from params that would modify the search ;; grab things from params that would modify the search
(define op (if (equal? (dict-ref params 'op #f) "or") '("or" . "OR") '("and" . "AND"))) (define op (if (equal? (dict-ref params 'op #f) "or") '("or" . "OR") '("and" . "AND")))
(define sort (if (equal? (dict-ref params 'sort #f) "len") '("len" . "len desc") '("relevance" . "score desc"))) (define sort (if (equal? (dict-ref params 'sort #f) "len") '("len" . "len desc") '("relevance" . "score desc")))
@ -36,8 +35,7 @@
("hl.tag.post" . "</mark>") ("hl.tag.post" . "</mark>")
("sort" . ,(cdr sort)))))) ("sort" . ,(cdr sort))))))
;; HTTP request to dest-url for search results ;; HTTP request to dest-url for search results
(log-outgoing dest-url) (define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5)))
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res)) (define json (easy:response-json res))
;; build result objects ;; build result objects

View file

@ -1,11 +0,0 @@
#lang typed/racket/base
(require "config.rkt")
(provide
; prints "out: <url>"
log-outgoing)
(: log-outgoing (String -> Void))
(define (log-outgoing url-string)
(when (config-true? 'log_outgoing)
(printf "out: ~a~n" url-string)))

View file

@ -1,74 +1,40 @@
"use strict"; "use strict";
let tabToFind = location.hash.length > 1 ? location.hash.substring(1) : null; const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null
for (let tabber of document.body.querySelectorAll(".wds-tabber")) {
let [tabs, contents] = getTabs(tabber);
for (let i in tabs) { for (const tabber of document.body.querySelectorAll(".wds-tabber")) {
let tab = tabs[i]; for (const [tab, content] of getTabberTabs(tabber)) {
let content = contents[i]; // set up click listener on every tab
tab.addEventListener("click", e => {
setCurrentTab(tabber, tab, content)
e.preventDefault()
})
tab.addEventListener("click", function(e) { // re-open a specific tab on page load based on the URL hash
setCurrentTab(tabber, tab, content); if (tab.dataset.hash === tabFromHash) {
e.preventDefault(); setCurrentTab(tabber, tab, content)
}); tab.scrollIntoView()
if (tab.dataset.hash === tabToFind) {
setCurrentTab(tabber, tab, content);
} }
} }
} }
document.body.classList.remove("bw-tabs-nojs");
function getTabberTabs(tabber) {
// need to scope the selector to handle nested tabs. see /unturned/wiki/Crate for an example
function getTabs(tabber) { const tabs = [...tabber.querySelectorAll(":scope > .wds-tabs__wrapper .wds-tabs__tab")]
let tabs = []; const contents = [...tabber.querySelectorAll(":scope > .wds-tab__content")]
let contents = []; return tabs.map((_, index) => [tabs[index], contents[index]]) // transpose arrays into [[tab, content], ...]
for (let i of tabber.querySelector(".wds-tabs__wrapper").querySelectorAll(".wds-tabs__tab")) {
tabs.push(i);
}
for (let i of tabber.children) {
if (!i.matches(".wds-tab__content")) {
continue;
}
contents.push(i);
}
return [tabs, contents];
}
function getCurrentTab(tabber) {
let tab = null;
let content = null;
tab = tabber.querySelector(".wds-tabs__wrapper").querySelector(".wds-tabs__tab.wds-is-current");
for (let i of tabber.children) {
if (!i.matches(".wds-tab__content.wds-is-current")) {
continue;
}
content = i;
break;
}
return [tab, content];
} }
function setCurrentTab(tabber, tab, content) { function setCurrentTab(tabber, tab, content) {
let [currentTab, currentContent] = getCurrentTab(tabber); // clear currently selected tab
if (currentTab) { getTabberTabs(tabber).flat().forEach(e => e.classList.remove("wds-is-current"))
currentTab.classList.remove("wds-is-current");
}
if (currentContent) {
currentContent.classList.remove("wds-is-current");
}
tab.classList.add("wds-is-current"); // select new tab
content.classList.add("wds-is-current"); tab.classList.add("wds-is-current")
content.classList.add("wds-is-current")
if (tab.dataset.hash) { if (tab.dataset.hash) {
let fragment = "#" + tab.dataset.hash; history.replaceState(null, "", `#${tab.dataset.hash}`)
if (location.hash !== fragment) {
history.pushState(null, "", fragment);
}
} }
} }
document.body.classList.remove("bw-tabs-nojs")