Compare commits

..

No commits in common. "main" and "main" have entirely different histories.
main ... main

16 changed files with 141 additions and 368 deletions

View file

@ -1,5 +1,6 @@
#lang racket/base
(require "../src/data.rkt"
(require (prefix-in easy: net/http-easy)
"../src/data.rkt"
"xexpr-utils.rkt")
(provide

View file

@ -22,6 +22,8 @@
(provide
; headers to always send on all http responses
always-headers
; timeout durations for http-easy requests
timeouts
; generates a consistent footer
application-footer
; generates a consistent template for wiki page content to sit in
@ -37,6 +39,7 @@
(define always-headers
(list (header #"Referrer-Policy" #"same-origin") ; header to not send referers to fandom
(header #"Link" (string->bytes/latin-1 link-header))))
(define timeouts (easy:make-timeout-config #:lease 5 #:connect 5))
(define-runtime-path path-static "../static")
(define theme-icons
@ -68,6 +71,9 @@
`(p ,(format "This instance is run by the ~a developer, " (config-get 'application_name))
(a (@ (href "https://cadence.moe/contact"))
"Cadence")
". Proudly hosted by "
(a (@ (href "http://alphamethyl.barr0w.net"))
"Barrow Network Solutions" (sup "XD"))
".")
`(p
,(format "This unofficial instance is based off the ~a source code, but is not controlled by the code developer." (config-get 'application_name)))))
@ -85,13 +91,11 @@
;; generate a notice with a link if a fandom wiki has a replacement as part of NIWA or similar
;; if the wiki has no replacement, display nothing
(define (extwiki-notice wikiname title req user-cookies)
(define (extwiki-notice wikiname title)
(define xt (findf (λ (item) (member wikiname (extwiki^-wikinames item))) extwikis))
(cond/var
[xt
(let* ([seen? (member wikiname (user-cookies^-notices user-cookies))]
[aside-class (if seen? "niwa__notice niwa--seen" "niwa__notice")]
[group (hash-ref extwiki-groups (extwiki^-group xt))]
(let* ([group (hash-ref extwiki-groups (extwiki^-group xt))]
[search-page (format "/Special:Search?~a"
(params->query `(("search" . ,title)
("go" . "Go"))))]
@ -102,24 +106,21 @@
[props (extwiki-props^ go)])
(cond
[(eq? (extwiki^-banner xt) 'default)
`(aside (@ (class ,aside-class))
`(aside (@ (class "niwa__notice"))
(h1 (@ (class "niwa__header")) ,(extwiki^-name xt) " has its own website separate from Fandom.")
(a (@ (class "niwa__go") (href ,go)) "Read " ,title " on " ,(extwiki^-name xt) "")
(div (@ (class "niwa__cols"))
(div (@ (class "niwa__left"))
(p ,((extwiki^-description xt) props))
(p ,((extwiki-group^-description group) props))
(p "This wiki's core community has largely migrated away from Fandom. You should "
(p ,((extwiki^-description xt) props))
(p "This wiki's core community has wholly migrated away from Fandom. You should "
(a (@ (href ,go)) "go to " ,(extwiki^-name xt) " now!"))
(p (@ (class "niwa__feedback"))
,@(add-between
`(,@(for/list ([link (extwiki-group^-links group)])
`(a (@ (href ,(cdr link))) ,(car link)))
"This notice is from BreezeWiki"
(a (@ (rel "nofollow")
(class "niwa__got-it")
(href ,(user-cookies-setter-url/add-notice req user-cookies wikiname)))
"OK, got it"))
(a (@ (href "https://docs.breezewiki.com/Reporting_Bugs.html")) "Feedback?"))
" / ")))
(div (@ (class "niwa__right"))
(img (@ (class "niwa__logo") (src ,(extwiki^-logo xt)))))))]
@ -174,7 +175,7 @@
(define styles
(list
(format "~a/wikia.php?controller=ThemeApi&method=themeVariables&variant=~a" origin (user-cookies^-theme user-cookies))
(format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.fandom.photoGallery.gallery.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin)))
(format "~a/load.php?lang=en&modules=site.styles%7Cskin.fandomdesktop.styles%7Cext.fandom.PortableInfoboxFandomDesktop.css%7Cext.fandom.GlobalComponents.CommunityHeaderBackground.css%7Cext.gadget.site-styles%2Csound-styles&only=styles&skin=fandomdesktop" origin)))
(if (config-true? 'strict_proxy)
(map u-proxy-url styles)
styles)]
@ -202,11 +203,10 @@
`(script (@ (type "module") (src ,(get-static-url "search-suggestions.js"))))
"")
(script (@ (type "module") (src ,(get-static-url "countdown.js"))))
(script (@ (defer) (src ,(get-static-url "tabs.js"))))
(link (@ (rel "icon") (href ,(u (λ (v) (config-true? 'strict_proxy))
(λ (v) (u-proxy-url v))
(head-data^-icon-url head-data))))))
(body (@ (class ,(head-data^-body-class head-data) " bw-tabs-nojs"))
(body (@ (class ,(head-data^-body-class head-data)))
,(let ([extension-eligible?
(cond/var
[(not req) #f]
@ -227,7 +227,7 @@
(div (@ (class "fandom-community-header__background tileHorizontally header")))
(div (@ (class "page"))
(main (@ (class "page__main"))
,(extwiki-notice wikiname title req user-cookies)
,(extwiki-notice wikiname title)
(div (@ (class "custom-top"))
(h1 (@ (class "page-title")) ,title)
(nav (@ (class "sitesearch"))

View file

@ -104,7 +104,7 @@
; all values here are optimised for maximum prettiness
(parameterize ([pretty-print-columns 80])
(display "config: ")
(pretty-write ((inst sort (Pairof Symbol String) Symbol)
(pretty-write ((inst sort (Pairof Symbol String))
(hash->list (make-immutable-hasheq combined-alist))
symbol<?
#:key car))))

View file

@ -1,15 +1,14 @@
#lang racket/base
(require racket/list
racket/match
racket/string
web-server/http/request-structs
net/url-string
(only-in net/cookies/server cookie-header->alist cookie->set-cookie-header make-cookie)
(prefix-in easy: net/http-easy)
db
memo
"fandom-request.rkt"
"static-data.rkt"
"whole-utils.rkt"
"../lib/url-utils.rkt"
"../lib/xexpr-utils.rkt"
"../archiver/archiver-database.rkt"
@ -28,8 +27,7 @@
user-cookies-getter
user-cookies-default
user-cookies-setter
user-cookies-setter-url
user-cookies-setter-url/add-notice)
user-cookies-setter-url)
(struct siteinfo^ (sitename basepage license) #:transparent)
(struct license^ (text url) #:transparent)
@ -56,21 +54,21 @@
(vector-ref row 3)))
siteinfo-default)]
[else
(define res
(fandom-get-api
wikiname
'(("action" . "query")
("meta" . "siteinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2"))))
(cond [(= (easy:response-status-code res) 200)
(define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))]
[else siteinfo-default])]))
(define dest-url
(format "https://~a.fandom.com/api.php?~a"
wikiname
(params->query '(("action" . "query")
("meta" . "siteinfo")
("siprop" . "general|rightsinfo")
("format" . "json")
("formatversion" . "2")))))
(log-outgoing dest-url)
(define res (easy:get dest-url))
(define data (easy:response-json res))
(siteinfo^ (jp "/query/general/sitename" data)
(second (regexp-match #rx"/wiki/(.*)" (jp "/query/general/base" data)))
(license^ (jp "/query/rightsinfo/text" data)
(jp "/query/rightsinfo/url" data)))]))
(define/memoize (head-data-getter wikiname) #:hash hash
;; data will be stored here, can be referenced by the memoized closure
@ -92,8 +90,8 @@
;; then no matter what, return the best information we have so far
this-data))
(struct user-cookies^ (theme notices) #:prefab)
(define user-cookies-default (user-cookies^ 'default '()))
(struct user-cookies^ (theme) #:prefab)
(define user-cookies-default (user-cookies^ 'default))
(define (user-cookies-getter req)
(define cookie-header (headers-assq* #"cookie" (request-headers/raw req)))
(define cookies-alist (if cookie-header (cookie-header->alist (header-value cookie-header) bytes->string/utf-8) null))
@ -102,29 +100,16 @@
(match pair
[(cons "theme" (and theme (or "light" "dark" "default")))
(values 'theme (string->symbol theme))]
[(cons "notices" notices)
(values 'notices (string-split notices "|"))]
[_ (values #f #f)])))
(user-cookies^
(hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default))
(hash-ref cookies-hash 'notices (user-cookies^-notices user-cookies-default))))
(hash-ref cookies-hash 'theme (user-cookies^-theme user-cookies-default))))
(define (user-cookies-setter user-cookies)
(map (λ (c) (header #"Set-Cookie" (cookie->set-cookie-header c)))
(list (make-cookie "theme" (symbol->string (user-cookies^-theme user-cookies))
#:path "/"
#:max-age (* 60 60 24 365 10))
(make-cookie "notices" (string-join (user-cookies^-notices user-cookies) "|")
#:path "/"
#:max-age (* 60 60 24 365 10)))))
(define (user-cookies-setter-url req new-settings)
(format "/set-user-settings?~a" (params->query `(("next_location" . ,(url->string (request-uri req)))
("new_settings" . ,(format "~s" new-settings))))))
(define (user-cookies-setter-url/add-notice req user-cookies notice-name)
(user-cookies-setter-url
req
(struct-copy user-cookies^ user-cookies
[notices (cons notice-name (user-cookies^-notices user-cookies))])))
("new_settings" . ,(format "~a" new-settings))))))

View file

@ -33,43 +33,26 @@
; don't forget that I'm returning *code* - return a call to the function
(datum->syntax stx `(make-dispatcher-tree ,ds)))
; guard that the page returned a response, otherwise print more detailed debugging information
(define-syntax-rule (page ds name)
(λ (req)
(define dispatcher (hash-ref ds (quote name)))
(define page-response (dispatcher req))
(if (response? page-response)
page-response
(response/output
#:code 500
#:mime-type #"text/plain"
(λ (out)
(for ([port (list (current-error-port) out)])
(parameterize ([current-output-port port])
(printf "error in ~a:~n expected page to return a response~n actually returned: ~v~n"
(quote name)
page-response))))))))
(define (make-dispatcher-tree ds)
(define subdomain-dispatcher (hash-ref ds 'subdomain-dispatcher))
(define tree
(sequencer:make
subdomain-dispatcher
(pathprocedure:make "/" (page ds page-home))
(pathprocedure:make "/proxy" (page ds page-proxy))
(pathprocedure:make "/search" (page ds page-global-search))
(pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
(pathprocedure:make "/" (hash-ref ds 'page-home))
(pathprocedure:make "/proxy" (hash-ref ds 'page-proxy))
(pathprocedure:make "/search" (hash-ref ds 'page-global-search))
(pathprocedure:make "/set-user-settings" (hash-ref ds 'page-set-user-settings))
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (hash-ref ds 'page-it-works))
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-category)))
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (hash-ref ds 'page-file)))
(if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki-offline)))
(λ (_conn _req) (next-dispatcher)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-wiki)))
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (hash-ref ds 'page-search)))
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (hash-ref ds 'redirect-wiki-home)))
(if (config-true? 'feature_offline::enabled)
(filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (page ds page-static-archive)))
(filter:make (pregexp (format "^/archive/~a/(styles|images)/.+$" px-wikiname)) (lift:make (hash-ref ds 'page-static-archive)))
(λ (_conn _req) (next-dispatcher)))
(hash-ref ds 'static-dispatcher)
(lift:make (hash-ref ds 'page-not-found))))

View file

@ -27,13 +27,6 @@
(λ (props)
`(p "The Square Enix Indpendent Wiki Alliance, or SEIWA, is a network of independent wikis established in 2011 and focused on providing high-quality coverage of Square Enix and its content. We work together, along with our affiliates and others, to co-operate and support one another while providing the best-quality content on the various Square Enix video games and media.")))
'GWN
(extwiki-group^
"GWN"
'(("Gaming Wiki Network" . "https://gamingwikinetwork.org/"))
(λ (props)
`(p "This wiki is part of the Gaming Wiki Network, a network of independently-hosted wikis about video game franchises. The GWN was founded on October 21, 2022. It aims to support all gaming communities in building independently-hosted wikis.")))
'Terraria
(extwiki-group^
"Terraria"
@ -78,25 +71,11 @@
(λ (props)
'(p "The wiki was founded by Citricsquid on July 16th, 2009 as a way to document information from Minecraft. Since November 15th, 2010, it has been hosted by Curse Media. On December 12th, 2018, it moved to Fandom as it purchased Curse Media. Since September 24, 2023, it forked from Fandom and has been hosted by Weird Gloop.")))
'Tardis
(extwiki-group^
"Tardis"
'(("Forking announcement" . "https://tardis.wiki/wiki/Tardis:Forking_announcement")
("Discussion on Reddit" . "https://old.reddit.com/r/doctorwho/comments/1azxmrl/tardis_wiki_has_regenerated/"))
(λ (props) '()))
'Rainverse
(extwiki-group^
"Rainverse"
'(("Forking announcement" . "https://transfem.social/notes/9qsqdkmqi78e01bh"))
(λ (props)
'()))
'empty
(extwiki-group^
"Misc"
'(("This wiki doesn't have a description yet. Add one?" . "https://docs.breezewiki.com/Reporting_Bugs.html"))
(λ (props) '()))))
#f)))
;; wikiname, niwa-name, url, logo-url
(struct extwiki^ (wikinames banner group name home logo description) #:transparent)
@ -330,11 +309,11 @@
(extwiki^
'("zelda" "zelda-archive") 'default
'NIWA
"Zelda Wiki"
"https://zeldawiki.wiki/wiki/Main_Page"
"Zeldapedia"
"https://zeldapedia.wiki/wiki/Main_Page"
"https://niwanetwork.org/images/logos/zeldapedia.png"
(λ (props)
`((p "Founded on April 23, 2005, Zelda Wiki is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zelda Wiki went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons."))))
`((p "Founded on April 23, 2005 as Zelda Wiki, today's Zeldapedia is your definitive source for encyclopedic information on The Legend of Zelda series, as well as all of the latest Zelda news. Zeldapedia went independent from Fandom in October 2022, citing Fandom's recent buyouts and staffing decisions among their reasons."))))
(extwiki^
'("chrono") 'default
@ -440,99 +419,6 @@
(λ (props)
`()))
(extwiki^
'("tardis") 'default
'Tardis
"TARDIS Wiki"
"https://tardis.wiki/wiki/Doctor_Who_Wiki"
"https://tardis.wiki/w/images/Tardis_Images/e/e6/Site-logo.png"
(λ (props)
`()))
(extwiki^
'("wizardry") 'default
'GWN
"Wizardry Wiki"
"https://wizardry.wiki.gg/wiki/Wizardry_Wiki"
"https://wizardry.wiki.gg/images/e/e6/Site-logo.png"
(λ (props)
`((p "On March 21, 2023, the wiki has decided to leave and abandoning from Fandom due to numerous of issues such as intrusive advertising, long-lasting bugs, restrictions on customization, etcetera. Wizardry Wiki was officially inducted into the wiki.gg wikifarm, with all contents forked over.")
(p "The wiki has partnered with " (a (@ (href "https://fallout.wiki/")) "Independent Fallout Wiki") " as of June 14, 2024."))))
(extwiki^
'("jackryan") 'default
'GWN
"Tom Clancy Wiki"
"https://tomclancy.wiki.gg/wiki/Tom_Clancy_Wiki"
"https://tomclancy.wiki.gg/images/thumb/c/c5/Jack_Ryan_Logo_Dark.png/600px-Jack_Ryan_Logo_Dark.png"
(λ (props)
`((p "The Tom Clancy Wiki is a collaborative encyclopedia dedicated to Tom Clancys franchises. The Tom Clancy franchise is a 40-year old expansive franchise founded by Tom Clancy, telling several unique sagas through books, video games, and films, as well as a TV show."))))
(extwiki^
'("hollowknight") 'default
'GWN
"Hollow Knight Wiki"
"https://hollowknight.wiki/wiki/Main_Page"
"https://gamingwikinetwork.org/images/logos/hollowknight.png"
(λ (props)
`((p "We are an independently hosted wiki for the games Hollow Knight and Hollow Knight: Silksong, created by fans, for fans. The wiki is a fork of the FANDOM Hollow Knight Wiki and was officially unveiled on October 31, 2023."))))
(extwiki^
'("hellokitty" "sanrio") 'default
'GWN
"Sanrio Wiki"
"https://sanriowiki.com/wiki/Sanrio_Wiki"
"https://cdn.sanriowiki.com/wiki.png"
(λ (props)
`((p "Sanrio Wiki is a project that was started on April 14, 2015 by EvieMelody. It was hosted on the wiki-farm ShoutWiki and has since become independent."))))
(extwiki^
'("sto") 'default
'GWN
"Star Trek Online Wiki"
"https://stowiki.net/wiki/Main_Page"
"https://gamingwikinetwork.org/images/logos/stowiki.png"
(λ (props)
`()))
(extwiki^
'("rayman-game" "ubisoftrayman") 'default
'GWN
"Rayman Wiki"
"https://raymanpc.com/wiki/en/Main_Page"
"https://raymanpc.com/wiki/script-en/resources/assets/logo-en.png?5c608"
(λ (props)
`()))
(extwiki^
'("granblue") 'empty
'empty
"Granblue Fantasy Wiki"
"https://gbf.wiki/"
"https://gbf.wiki/images/1/18/Vyrnball.png?0704c"
(λ (props)
`()))
(extwiki^
'("hellmet-roblox") 'empty
'empty
"HELLMET Wiki"
"https://hellmet.miraheze.org/wiki/Main_Page"
"https://static.miraheze.org/hellmetwiki/thumb/c/ce/Hellmet_Wiki_Logo.png/135px-Hellmet_Wiki_Logo.png"
(λ (props)
`()))
(extwiki^
'("rain-web-comic") 'default
'empty
"Rainverse Wiki"
"https://rainverse.wiki/wiki/Main_Page"
"https://static.miraheze.org/rainversewiki/2/2c/Rain_comic_cover.png"
(λ (props)
`((p "We have a newly-migrated Rainverse Wiki which escaped from Fandom! Rain is the comic that helped me figure out my gender, so I am really glad to have a wiki on a non-evil host.")
(p "Please stop using the abandoned copy of Rain Wiki on Fandom. Fandom is still \"training\" a generator which adds procedurally-generated bullshit to articles, with no way for users to remove or correct it, and they're demanding volunteer wiki admins waste time \"vetting\" the procedurally-generated BS for accuracy. As Jocelyn herself said, \"fuck Fandom forever.\"")
(p "If you are interested, please add more articles related to other Rainverse stories."))))
;; fandom wikinames * empty * empty * Name * Home Page
(extwiki^ '("aether") 'empty 'empty "Aether Wiki" "https://aether.wiki.gg/wiki/Aether_Wiki" #f #f)
(extwiki^ '("before-darkness-falls") 'empty 'empty "Before Darkness Falls Wiki" "https://beforedarknessfalls.wiki.gg/wiki/Before_Darkness_Falls_Wiki" #f #f)
@ -548,7 +434,6 @@
(extwiki^ '("doom") 'empty 'empty "DoomWiki.org" "https://doomwiki.org/wiki/Entryway" #f #f)
(extwiki^ '("dreamscaper") 'empty 'empty "Official Dreamscaper Wiki" "https://dreamscaper.wiki.gg/wiki/Dreamscaper_Wiki" #f #f)
(extwiki^ '("elderscrolls") 'empty 'empty "UESP" "https://en.uesp.net/wiki/Main_Page" #f #f)
(extwiki^ '("enterthegungeon" "exit-the-gungeon" "enter-the-gungeon-archive") 'empty 'empty "Official Enter The Gungeon Wiki" "https://enterthegungeon.wiki.gg/wiki/Enter_the_Gungeon_Wiki" "https://enterthegungeon.wiki.gg/images/e/e6/Site-logo.png" #f)
(extwiki^ '("fiend-folio") 'empty 'empty "Official Fiend Folio Wiki" "https://fiendfolio.wiki.gg/wiki/Fiend_Folio_Wiki" #f #f)
(extwiki^ '("foxhole") 'empty 'empty "Foxhole Wiki" "https://foxhole.wiki.gg/wiki/Foxhole_Wiki" #f #f)
(extwiki^ '("have-a-nice-death") 'empty 'empty "Have a Nice Death Wiki" "https://haveanicedeath.wiki.gg/wiki/Have_a_Nice_Death_Wiki" #f #f)

View file

@ -1,48 +0,0 @@
#lang typed/racket/base
(require "config.rkt"
"../lib/url-utils.rkt")
(define-type Headers (HashTable Symbol (U Bytes String)))
(require/typed net/http-easy
[#:opaque Timeout-Config timeout-config?]
[#:opaque Response response?]
[#:opaque Session session?]
[current-session (Parameter Session)]
[make-timeout-config ([#:lease Positive-Real] [#:connect Positive-Real] -> Timeout-Config)]
[get ((U Bytes String)
[#:close? Boolean]
[#:headers Headers]
[#:timeouts Timeout-Config]
[#:max-attempts Exact-Positive-Integer]
[#:max-redirects Exact-Nonnegative-Integer]
[#:user-agent (U Bytes String)]
-> Response)])
(provide
fandom-get
fandom-get-api
timeouts)
(define timeouts (make-timeout-config #:lease 5 #:connect 5))
(: no-headers Headers)
(define no-headers '#hasheq())
(: fandom-get (String String [#:headers (Option Headers)] -> Response))
(define (fandom-get wikiname path #:headers [headers #f])
(define dest-url (string-append "https://www.fandom.com" path))
(define host (string-append wikiname ".fandom.com"))
(log-outgoing wikiname path)
(get dest-url
#:timeouts timeouts
#:headers (hash-set (or headers no-headers) 'Host host)))
(: fandom-get-api (String (Listof (Pair String String)) [#:headers (Option Headers)] -> Response))
(define (fandom-get-api wikiname params #:headers [headers #f])
(fandom-get wikiname
(string-append "/api.php?" (params->query params))
#:headers headers))
(: log-outgoing (String String -> Void))
(define (log-outgoing wikiname path)
(when (config-true? 'log_outgoing)
(printf "out: ~a ~a~n" wikiname path)))

View file

@ -15,11 +15,11 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"page-wiki.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -73,24 +73,30 @@
(define-values (members-data page-data siteinfo)
(thread-values
(λ ()
(easy:response-json
(fandom-get-api
wikiname
`(("action" . "query")
("list" . "categorymembers")
("cmtitle" . ,prefixed-category)
("cmlimit" . "max")
("formatversion" . "2")
("format" . "json")))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "categorymembers")
("cmtitle" . ,prefixed-category)
("cmlimit" . "max")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(easy:response-json
(fandom-get-api
wikiname
`(("action" . "parse")
("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "parse")
("page" . ,prefixed-category)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(siteinfo-fetch wikiname))))

View file

@ -15,11 +15,11 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"page-wiki.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide page-file)
@ -40,7 +40,8 @@
(imageDescription . #f))))
(define (url-content-type url)
(define dest-res (easy:head url))
(log-outgoing url)
(define dest-res (easy:head url #:timeouts timeouts))
(easy:response-headers-ref dest-res 'content-type))
(define (get-media-html url content-type)
@ -105,18 +106,20 @@
(response-handler
(define wikiname (path/param-path (first (url-path (request-uri req)))))
(define prefixed-title (path/param-path (caddr (url-path (request-uri req)))))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname prefixed-title))
(define origin (format "https://~a.fandom.com" wikiname))
(define source-url (format "~a/wiki/~a" origin prefixed-title))
(define-values (media-detail siteinfo)
(thread-values
(λ ()
(define dest-res
(fandom-get
wikiname
(format "/wikia.php?~a"
(params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title))))))
(define dest-url
(format "~a/wikia.php?~a"
origin
(params->query `(("format" . "json") ("controller" . "Lightbox")
("method" . "getMediaDetail")
("fileTitle" . ,prefixed-title)))))
(log-outgoing dest-url)
(define dest-res (easy:get dest-url #:timeouts timeouts))
(easy:response-json dest-res))
(λ ()
(siteinfo-fetch wikiname))))

View file

@ -2,6 +2,7 @@
(require racket/dict
racket/list
racket/string
(prefix-in easy: net/http-easy)
; html libs
html-writing
; web server libs
@ -17,6 +18,7 @@
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide

View file

@ -17,12 +17,12 @@
"application-globals.rkt"
"config.rkt"
"data.rkt"
"fandom-request.rkt"
"../lib/pure-utils.rkt"
"../lib/syntax.rkt"
"../lib/thread-utils.rkt"
"../lib/tree-updater.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -38,20 +38,25 @@
(define (page-wiki req)
(define wikiname (path/param-path (first (url-path (request-uri req)))))
(define user-cookies (user-cookies-getter req))
(define origin (format "https://~a.fandom.com" wikiname))
(define path (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
(define-values (dest-res siteinfo)
(thread-values
(λ ()
(fandom-get-api
wikiname
`(("action" . "parse")
("page" . ,path)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json"))
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "parse")
("page" . ,path)
("prop" . "text|headhtml|langlinks")
("formatversion" . "2")
("format" . "json")))))
(log-outgoing dest-url)
(easy:get dest-url
#:timeouts timeouts
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies))))))
(λ ()
(siteinfo-fetch wikiname))))
@ -98,13 +103,4 @@
#:code 200
#:headers headers
(λ (out)
(write-html body out))))))]
[(eq? 404 (easy:response-status-code dest-res))
(next-dispatcher)]
[else
(response-handler
(error 'page-wiki "Tried to load page ~a/~v~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
wikiname
path
(easy:response-status-code dest-res)
(easy:response-body dest-res)))]))
(write-html body out))))))]))

View file

@ -3,8 +3,8 @@
(prefix-in easy: net/http-easy)
"application-globals.rkt"
"config.rkt"
"fandom-request.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -17,14 +17,20 @@
'(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
(define (search-fandom wikiname query params)
(define res
(fandom-get-api
wikiname
`(("action" . "query")
("list" . "search")
("srsearch" . ,query)
("formatversion" . "2")
("format" . "json"))))
;; constructing the URL where I want to get fandom data from...
(define origin (format "https://~a.fandom.com" wikiname))
;; the dest-URL will look something like https://minecraft.fandom.com/api.php?action=query&list=search&srsearch=Spawner&formatversion=2&format=json
(define dest-url
(format "~a/api.php?~a"
origin
(params->query `(("action" . "query")
("list" . "search")
("srsearch" . ,query)
("formatversion" . "2")
("format" . "json")))))
;; HTTP request to dest-url for search results
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res))
(define search-results (jp "/query/search" json))
(generate-results-content-fandom wikiname query search-results))

View file

@ -5,6 +5,7 @@
"application-globals.rkt"
"../lib/html-parsing/main.rkt"
"../lib/url-utils.rkt"
"whole-utils.rkt"
"../lib/xexpr-utils.rkt")
(provide
@ -35,7 +36,8 @@
("hl.tag.post" . "</mark>")
("sort" . ,(cdr sort))))))
;; HTTP request to dest-url for search results
(define res (easy:get dest-url #:timeouts (easy:make-timeout-config #:lease 5 #:connect 5)))
(log-outgoing dest-url)
(define res (easy:get dest-url #:timeouts timeouts))
(define json (easy:response-json res))
;; build result objects

11
src/whole-utils.rkt Normal file
View file

@ -0,0 +1,11 @@
#lang typed/racket/base
(require "config.rkt")
(provide
; prints "out: <url>"
log-outgoing)
(: log-outgoing (String -> Void))
(define (log-outgoing url-string)
(when (config-true? 'log_outgoing)
(printf "out: ~a~n" url-string)))

View file

@ -202,11 +202,11 @@ figcaption, .lightbox-caption, .thumbcaption {
padding: 0;
}
/* show tabs if tabs.js isn't loaded */
.bw-tabs-nojs .wds-tabs__wrapper {
/* show tabs always */
.wds-tabs__wrapper {
display: none;
}
.bw-tabs-nojs .wds-tab__content {
.wds-tab__content {
display: block;
}
@ -431,25 +431,6 @@ a.ext-audiobutton { /* see hearthstone/wiki/Diablo_(Duels_hero) */
font-size: 14px;
text-align: right;
}
/* more compact notice after it's been seen the first time */
.niwa--seen {
padding: 1.5vw 2vw 2vw;
overflow-y: auto;
max-height: min(280px, 33vh);
font-size: 17px;
margin-top: -2vw;
margin-bottom: 12px;
}
.niwa--seen .niwa__header {
font-size: 26px;
}
.niwa--seen .niwa__go {
padding: 10px 18px;
font-size: 20px;
}
.niwa--seen .niwa__got-it {
display: none;
}
/* media queries */

View file

@ -1,40 +0,0 @@
"use strict";
const tabFromHash = location.hash.length > 1 ? location.hash.substring(1) : null
for (const tabber of document.body.querySelectorAll(".wds-tabber")) {
for (const [tab, content] of getTabberTabs(tabber)) {
// set up click listener on every tab
tab.addEventListener("click", e => {
setCurrentTab(tabber, tab, content)
e.preventDefault()
})
// re-open a specific tab on page load based on the URL hash
if (tab.dataset.hash === tabFromHash) {
setCurrentTab(tabber, tab, content)
tab.scrollIntoView()
}
}
}
function getTabberTabs(tabber) {
// need to scope the selector to handle nested tabs. see /unturned/wiki/Crate for an example
const tabs = [...tabber.querySelectorAll(":scope > .wds-tabs__wrapper .wds-tabs__tab")]
const contents = [...tabber.querySelectorAll(":scope > .wds-tab__content")]
return tabs.map((_, index) => [tabs[index], contents[index]]) // transpose arrays into [[tab, content], ...]
}
function setCurrentTab(tabber, tab, content) {
// clear currently selected tab
getTabberTabs(tabber).flat().forEach(e => e.classList.remove("wds-is-current"))
// select new tab
tab.classList.add("wds-is-current")
content.classList.add("wds-is-current")
if (tab.dataset.hash) {
history.replaceState(null, "", `#${tab.dataset.hash}`)
}
}
document.body.classList.remove("bw-tabs-nojs")