From 68a5ac20b640a9c2202707805e4c60a4f2b3f7af Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 6 Jun 2020 04:39:22 +0200 Subject: [PATCH] Proxy media instead of using file cache --- src/nitter.nim | 2 +- src/routes/media.nim | 128 +++++++++++++++++++++++++----------------- src/routes/status.nim | 2 +- src/tokens.nim | 2 +- src/utils.nim | 3 - src/views/rss.nimf | 2 +- src/views/tweet.nim | 2 +- 7 files changed, 80 insertions(+), 61 deletions(-) diff --git a/src/nitter.nim b/src/nitter.nim index 08c5372..f4eb8a2 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -1,4 +1,4 @@ -import asyncdispatch, mimetypes +import asyncdispatch from net import Port import jester diff --git a/src/routes/media.nim b/src/routes/media.nim index b0639ea..c86a786 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -1,4 +1,5 @@ -import asyncfile, uri, strutils, httpclient, os, mimetypes +import uri, strutils, httpclient, os, hashes +import asynchttpserver, asyncstreams, asyncfile, asyncnet import jester, regex @@ -6,12 +7,59 @@ import router_utils import ".."/[types, formatters, agents] import ../views/general -export asyncfile, httpclient, os, strutils -export regex +export asynchttpserver, asyncstreams, asyncfile, asyncnet +export httpclient, os, strutils, asyncstreams, regex + +const + m3u8Regex* = re"""url="(.+.m3u8)"""" + m3u8Mime* = "application/vnd.apple.mpegurl" + maxAge* = "max-age=604800" -const m3u8Regex* = re"""url="(.+.m3u8)"""" let mediaAgent* = getAgent() +template respond*(req: asynchttpserver.Request; headers) = + var msg = "HTTP/1.1 200 OK\c\L" + for k, v in headers: + msg.add(k & ": " & v & "\c\L") + + msg.add "\c\L" + yield req.client.send(msg) + +proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = + result = Http200 + let + request = req.getNativeReq() + client = newAsyncHttpClient(userAgent=mediaAgent) + + try: + let res = await client.get(url) + if res.status != "200 OK": + return Http404 + + let hashed = $hash(url) + if request.headers.getOrDefault("If-None-Match") == hashed: + return Http304 + + let headers = newHttpHeaders({ + "Content-Type": res.headers["content-type", 0], + "Content-Length": res.headers["content-length", 0], + "Cache-Control": maxAge, + "ETag": hashed + }) + + respond(request, headers) + + var (hasValue, data) = (true, "") + while hasValue: + (hasValue, data) = await res.bodyStream.read() + if hasValue: + await request.client.send(data) + data.setLen 0 + except HttpRequestError, OSError: + result = Http404 + finally: + client.safeClose() + proc createMediaRouter*(cfg: Config) = router media: get "/pic/?": @@ -24,48 +72,13 @@ proc createMediaRouter*(cfg: Config) = let uri = parseUri(decodeUrl(@"url")) cond isTwitterUrl($uri) == true - let path = uri.path.split("/")[2 .. ^1].join("/") - let filename = cfg.cacheDir / cleanFilename(path & uri.query) - - if path.len == 0: - resp Http404 - - if not existsDir(cfg.cacheDir): - createDir(cfg.cacheDir) - - if not existsFile(filename): - let client = newAsyncHttpClient(userAgent=mediaAgent) - var failed = false - - try: - await client.downloadFile($uri, filename) - except HttpRequestError: - failed = true - removeFile(filename) - except OSError as e: - echo "Disk full, or network error: ", e.msg - failed = true - finally: - client.safeClose() - - if failed: - resp Http404 - - sendFile(filename) - - get "/gif/@url": - cond "http" in @"url" - cond "twimg" in @"url" - cond "mp4" in @"url" or "gif" in @"url" - - let url = decodeUrl(@"url") - cond isTwitterUrl(url) == true - - let content = await safeFetch(url, mediaAgent) - if content.len == 0: resp Http404 - - let filename = parseUri(url).path.split(".")[^1] - resp content, settings.mimes.getMimetype(filename) + enableRawMode() + let code = await proxyMedia(request, $uri) + if code == Http200: + enableRawMode() + break route + else: + resp code get "/video/@sig/@url": cond "http" in @"url" @@ -75,17 +88,26 @@ proc createMediaRouter*(cfg: Config) = if getHmac(url) != @"sig": resp showError("Failed to verify signature", cfg) - var content = await safeFetch(url, mediaAgent) - if content.len == 0: resp Http404 + if ".mp4" in url or ".ts" in url: + let code = await proxyMedia(request, url) + if code == Http200: + enableRawMode() + break route + else: + resp code + var content: string if ".vmap" in url: var m: RegexMatch - discard content.find(m3u8Regex, m) - url = decodeUrl(content[m.group(0)[0]]) content = await safeFetch(url, mediaAgent) + if content.find(m3u8Regex, m): + url = decodeUrl(content[m.group(0)[0]]) + content = await safeFetch(url, mediaAgent) + else: + resp Http404 if ".m3u8" in url: - content = proxifyVideo(content, prefs.proxyVideos) + let vid = await safeFetch(url, mediaAgent) + content = proxifyVideo(vid, prefs.proxyVideos) - let ext = parseUri(url).path.split(".")[^1] - resp content, settings.mimes.getMimetype(ext) + resp content, m3u8Mime diff --git a/src/routes/status.nim b/src/routes/status.nim index 681c9b2..6919967 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -45,7 +45,7 @@ proc createStatusRouter*(cfg: Config) = video = getVideoEmbed(cfg, conv.tweet.id) elif conv.tweet.gif.isSome(): images = @[get(conv.tweet.gif).thumb] - video = getGifUrl(get(conv.tweet.gif).url) + video = getPicUrl(get(conv.tweet.gif).url) let html = renderConversation(conv, prefs, getPath() & "#m") resp renderMain(html, request, cfg, title, desc, diff --git a/src/tokens.nim b/src/tokens.nim index 7212350..afbf2f7 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -35,7 +35,7 @@ proc expired(token: Token): bool {.inline.} = result = token.init < getTime() - expirationTime proc isLimited(token: Token): bool {.inline.} = - token == nil or token.remaining <= 1 and token.reset > getTime() or + token == nil or (token.remaining <= 1 and token.reset > getTime()) or token.expired proc release*(token: Token) = diff --git a/src/utils.nim b/src/utils.nim index b2bdb97..00b9a4c 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -28,9 +28,6 @@ proc getVidUrl*(link: string): string = url = encodeUrl(link) &"/video/{sig}/{url}" -proc getGifUrl*(link: string): string = - &"/gif/{encodeUrl(link)}" - proc getPicUrl*(link: string): string = &"/pic/{encodeUrl(link)}" diff --git a/src/views/rss.nimf b/src/views/rss.nimf index 1857d8c..727cd3d 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -36,7 +36,7 @@ #elif tweet.gif.isSome: #let thumb = &"https://{hostname}{getPicUrl(get(tweet.gif).thumb)}" -#let url = &"https://{hostname}{getGifUrl(get(tweet.gif).url)}" +#let url = &"https://{hostname}{getPicUrl(get(tweet.gif).url)}" #end if diff --git a/src/views/tweet.nim b/src/views/tweet.nim index e8a6fbc..e1dc01e 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -107,7 +107,7 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode = tdiv(class="gallery-gif", style={maxHeight: "unset"}): tdiv(class="attachment"): let thumb = getPicUrl(gif.thumb) - let url = getGifUrl(gif.url) + let url = getPicUrl(gif.url) if prefs.autoplayGifs: video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""): source(src=url, `type`="video/mp4")