forked from cadence/breezewiki
Archiver now saves redirects
This commit is contained in:
parent
27c9680f5b
commit
a57445abcb
8 changed files with 197 additions and 121 deletions
|
|
@ -377,7 +377,7 @@
|
|||
'("runescape") 'default
|
||||
'RuneScape
|
||||
"RuneScape Wiki"
|
||||
"https://runescape.wiki/"
|
||||
"https://runescape.wiki/w/Main_Page"
|
||||
"https://runescape.wiki/images/Wiki.png"
|
||||
(λ (props)
|
||||
`((p "The RuneScape Wiki was founded on April 8, 2005. In October 2018, the wiki left Fandom (then Wikia), citing their apathy towards the wiki and excessive advertisements."))))
|
||||
|
|
@ -386,7 +386,7 @@
|
|||
'("oldschoolrunescape") 'default
|
||||
'RuneScape
|
||||
"Old School RuneScape Wiki"
|
||||
"https://oldschool.runescape.wiki/"
|
||||
"https://oldschool.runescape.wiki/w/Main_Page"
|
||||
"https://oldschool.runescape.wiki/images/Wiki.png"
|
||||
(λ (props)
|
||||
`((p "The Old School RuneScape Wiki was founded on February 14, 2013. In October 2018, the RuneScape Wiki left Fandom (then Wikia), citing their apathy towards the wiki and excessive advertisements, with the Old School RuneScape Wiki following suit."))))
|
||||
|
|
@ -395,7 +395,7 @@
|
|||
'("runescapeclassic") 'default
|
||||
'RuneScape
|
||||
"RuneScape Classic Wiki"
|
||||
"https://classic.runescape.wiki/"
|
||||
"https://classic.runescape.wiki/w/Main_Page"
|
||||
"https://classic.runescape.wiki/images/Wiki.png"
|
||||
(λ (props)
|
||||
`((p "The Old School RuneScape Wiki was founded on April 19, 2009. In October 2018, the RuneScape Wiki left Fandom (then Wikia), citing their apathy towards the wiki and excessive advertisements, with the RuneScape Classic Wiki following suit."))))
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
[(not wikiname)
|
||||
(response/output
|
||||
#:code 400
|
||||
#:mime-type "text/plain"
|
||||
#:mime-type #"text/plain"
|
||||
(λ (out)
|
||||
(displayln "Requires wikiname and q parameters." out)))]
|
||||
[(or (not q) (equal? q ""))
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@
|
|||
;; grab the part after ?q= which is the search terms
|
||||
(define query (dict-ref params 'q #f))
|
||||
;; figure out which search provider we're going to use
|
||||
(define search-provider (hash-ref search-providers (config-get 'feature_offline::search)))
|
||||
(define search-provider (hash-ref search-providers (config-get 'feature_offline::search)
|
||||
(λ () (error 'search-provider "unknown search provider configured"))))
|
||||
|
||||
;; external special:search url to link at the bottom of the page as the upstream source
|
||||
(define external-search-url
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
web-server/dispatchers/dispatch
|
||||
; my libs
|
||||
"application-globals.rkt"
|
||||
"../archiver/archiver-database.rkt"
|
||||
"config.rkt"
|
||||
"data.rkt"
|
||||
"log.rkt"
|
||||
|
|
@ -40,6 +41,9 @@
|
|||
|
||||
(define path-archive (anytime-path ".." "storage/archive"))
|
||||
|
||||
(when (config-true? 'feature_offline::only)
|
||||
(void (get-slc)))
|
||||
|
||||
(define (page-wiki-offline req)
|
||||
(response-handler
|
||||
(define wikiname (path/param-path (first (url-path (request-uri req)))))
|
||||
|
|
@ -64,84 +68,94 @@
|
|||
[else (error 'archive-format "unknown archive format configured")]))
|
||||
(define fs-path (build-path path-archive wikiname (format (car archive-format) maybe-hashed-basename)))
|
||||
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname (basename->name-for-query basename)))
|
||||
(cond
|
||||
[(not (file-exists? fs-path))
|
||||
(unless (config-true? 'feature_offline::only)
|
||||
(next-dispatcher))
|
||||
(define mirror-path (url->string (request-uri req)))
|
||||
(cond/var
|
||||
|
||||
[(file-exists? fs-path)
|
||||
(when (config-true? 'debug)
|
||||
(printf "using offline mode for ~v~n" fs-path))
|
||||
(response-handler
|
||||
(define data (with-input-from-file fs-path (cdr archive-format)))
|
||||
(define article-title (jp "/parse/title" data))
|
||||
(define original-page (html->xexp (preprocess-html-wiki (jp "/parse/text" data))))
|
||||
(define page ((query-selector (λ (t a c) (has-class? "mw-parser-output" a)) original-page)))
|
||||
(define initial-head-data ((head-data-getter wikiname) data))
|
||||
(define head-data
|
||||
(case theme
|
||||
[(light dark)
|
||||
(struct-copy head-data^ initial-head-data
|
||||
[body-class (regexp-replace #rx"(theme-fandomdesktop-)(light|dark)"
|
||||
(head-data^-body-class initial-head-data)
|
||||
(format "\\1~a" theme))])]
|
||||
[else initial-head-data]))
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
`(div (@ (class "unsaved-page"))
|
||||
(style ".unsaved-page a { text-decoration: underline !important }")
|
||||
(p "breezewiki.com doesn't have this page saved.")
|
||||
(p "You can see this page by visiting a BreezeWiki mirror:")
|
||||
(ul
|
||||
(li (a (@ (href ,(format "https://antifandom.com~a" mirror-path))) "View on antifandom.com"))
|
||||
(li (a (@ (href ,(format "https://bw.artemislena.eu~a" mirror-path))) "View on artemislena.eu"))
|
||||
(li (a (@ (href ,source-url)) "or, you can see the original page on Fandom (ugh)")))
|
||||
(p "If you'd like " ,wikiname ".fandom.com to be added to breezewiki.com, " (a (@ (href "https://lists.sr.ht/~cadence/breezewiki-requests")) "let me know about it!")))
|
||||
(update-tree-wiki page wikiname)
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title (url-segments->guess-title segments)
|
||||
#:title article-title
|
||||
#:online-styles #f
|
||||
#:head-data head-data
|
||||
#:siteinfo (siteinfo-fetch wikiname)
|
||||
))
|
||||
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
|
||||
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
|
||||
(define headers
|
||||
(build-headers
|
||||
always-headers
|
||||
; redirect-query-parameter: only the string "no" is significant:
|
||||
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
|
||||
(when (and redirect-msg
|
||||
(not (equal? redirect-query-parameter "no")))
|
||||
(let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))]
|
||||
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
|
||||
(header #"Refresh" value)))))
|
||||
(when (config-true? 'debug)
|
||||
; used for its side effects
|
||||
; convert to string with error checking, error will be raised if xexp is invalid
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers always-headers
|
||||
#:headers headers
|
||||
(λ (out)
|
||||
(write-html body out)))]
|
||||
[#t
|
||||
(when (config-true? 'debug)
|
||||
(printf "using offline mode for ~v~n" fs-path))
|
||||
(response-handler
|
||||
(define data (with-input-from-file fs-path (cdr archive-format)))
|
||||
(define article-title (jp "/parse/title" data))
|
||||
(define original-page (html->xexp (preprocess-html-wiki (jp "/parse/text" data))))
|
||||
(define page ((query-selector (λ (t a c) (has-class? "mw-parser-output" a)) original-page)))
|
||||
(define initial-head-data ((head-data-getter wikiname) data))
|
||||
(define head-data
|
||||
(case theme
|
||||
[(light dark)
|
||||
(struct-copy head-data^ initial-head-data
|
||||
[body-class (regexp-replace #rx"(theme-fandomdesktop-)(light|dark)"
|
||||
(head-data^-body-class initial-head-data)
|
||||
(format "\\1~a" theme))])]
|
||||
[else initial-head-data]))
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
(update-tree-wiki page wikiname)
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title article-title
|
||||
#:online-styles #f
|
||||
#:head-data head-data
|
||||
#:siteinfo (siteinfo-fetch wikiname)
|
||||
))
|
||||
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
|
||||
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
|
||||
(define headers
|
||||
(build-headers
|
||||
always-headers
|
||||
; redirect-query-parameter: only the string "no" is significant:
|
||||
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
|
||||
(when (and redirect-msg
|
||||
(not (equal? redirect-query-parameter "no")))
|
||||
(let* ([dest (get-attribute 'href (bits->attributes ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))))]
|
||||
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
|
||||
(header #"Refresh" value)))))
|
||||
(when (config-true? 'debug)
|
||||
; used for its side effects
|
||||
; convert to string with error checking, error will be raised if xexp is invalid
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers headers
|
||||
(λ (out)
|
||||
(write-html body out))))])))
|
||||
(write-html body out))))]
|
||||
|
||||
;; page not found on disk, perhaps it's a redirect? redirects are stored in the database
|
||||
(var target (query-maybe-value* "select redirect from page where wikiname = ? and basename = ?" wikiname basename))
|
||||
[target
|
||||
(generate-redirect (basename->name-for-query target))]
|
||||
|
||||
;; breezewiki doesn't have the page archived, see if we can make a network request for it
|
||||
[(not (config-true? 'feature_offline::only))
|
||||
(next-dispatcher)]
|
||||
|
||||
;; no possible way to provide the page
|
||||
[else
|
||||
(define mirror-path (url->string (request-uri req)))
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
`(div (@ (class "unsaved-page"))
|
||||
(style ".unsaved-page a { text-decoration: underline !important }")
|
||||
(p "breezewiki.com doesn't have this page saved.")
|
||||
(p "You can see this page by visiting a BreezeWiki mirror:")
|
||||
(ul
|
||||
(li (a (@ (href ,(format "https://antifandom.com~a" mirror-path))) "View on antifandom.com"))
|
||||
(li (a (@ (href ,(format "https://bw.artemislena.eu~a" mirror-path))) "View on artemislena.eu"))
|
||||
(li (a (@ (href ,source-url)) "or, you can see the original page on Fandom (ugh)")))
|
||||
(p "If you'd like " ,wikiname ".fandom.com to be added to breezewiki.com, " (a (@ (href "https://lists.sr.ht/~cadence/breezewiki-requests")) "let me know about it!")))
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title (url-segments->guess-title segments)
|
||||
#:online-styles #f
|
||||
#:siteinfo (siteinfo-fetch wikiname)
|
||||
))
|
||||
(when (config-true? 'debug)
|
||||
; used for its side effects
|
||||
; convert to string with error checking, error will be raised if xexp is invalid
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers always-headers
|
||||
(λ (out)
|
||||
(write-html body out)))])))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue