blankie/breezewiki
1
0
Fork 0
forked from cadence/breezewiki
Code Issues Pull requests Projects Releases Packages Wiki Activity

Split massive uploads in Solr indexer

Browse source
This commit is contained in:
Cadence Ember 2024-01-11 22:33:59 +13:00
parent b02e2a4053
commit a52d131b93
1 changed files with 28 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
archiver/fts.rkt
View file

@ -8,6 +8,7 @@
racket/promise racket/promise
racket/port racket/port
racket/runtime-path racket/runtime-path
racket/sequence
racket/string racket/string
file/gunzip file/gunzip
db db
@ -156,8 +157,16 @@
(define data (define data
(cond (cond
[(and (read-from-cache?) (file-exists? "cache.rkt")) [(and (read-from-cache?) (file-exists? "cache.rkt"))
(displayln "Reading in...") (define size (file-size "cache.rkt"))
(with-input-from-file "cache.rkt" (λ () (read)))] (call-with-input-file "cache.rkt"
(λ (in)
(define quit (make-progress (λ () (progress^ (ceiling (/ (file-position in) 64 1024))
(ceiling (/ size 64 1024))
"Reading in..."))
2))
(begin0
(read in)
(quit))))]
[else [else
(define x (box (progress^ 0 1 "..."))) (define x (box (progress^ 0 1 "...")))
(define quit (make-progress (λ () (unbox x)))) (define quit (make-progress (λ () (unbox x))))
@ -183,11 +192,15 @@
(display "Converting... ") (display "Converting... ")
(flush-output) (flush-output)
(define ser (jsexpr->bytes data)) (define slice-size 30000)
(define slices (ceiling (/ (length data) slice-size)))
(for ([slice (in-slice slice-size data)]
[i (in-naturals 1)])
(define ser (jsexpr->bytes slice))
(define ser-port (open-input-bytes ser)) (define ser-port (open-input-bytes ser))
(define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024)) (define quit (make-progress (λ () (progress^ (ceiling (/ (file-position ser-port) 64 1024))
(ceiling (/ (bytes-length ser) 64 1024)) (ceiling (/ (bytes-length ser) 64 1024))
"Posting...")) (format "Posting... (~a/~a)" i slices)))
2)) 2))
(define res (define res
(post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname) (post (format "http://localhost:8983/solr/~a/update?commit=true" wikiname)
@ -195,6 +208,6 @@
#:headers '#hasheq((Content-Type . "application/json")) #:headers '#hasheq((Content-Type . "application/json"))
#:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300))) #:timeouts (make-timeout-config #:lease 5 #:connect 5 #:request 300)))
(quit) (quit)
(displayln (response-status-line res))) (displayln (response-status-line res))))
(run start) (run start)