Refactor Utility methods and migrate to OkHttp (#179)
This commit is contained in:
Kavin
GitHub
parent
b2cf84b41f
commit
c07cf5fd1f
|
|
@ -38,6 +38,9 @@ dependencies {
|
|||
implementation 'com.zaxxer:HikariCP:5.0.1'
|
||||
implementation 'org.springframework.security:spring-security-crypto:5.6.1'
|
||||
implementation 'commons-logging:commons-logging:1.2'
|
||||
implementation(platform("com.squareup.okhttp3:okhttp-bom:4.9.3"))
|
||||
implementation("com.squareup.okhttp3:okhttp")
|
||||
implementation("com.squareup.okhttp3:okhttp-brotli")
|
||||
}
|
||||
|
||||
shadowJar {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.2-bin.zip
|
||||
distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.3.3-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
|
|||
|
|
@ -1,28 +1,8 @@
|
|||
package me.kavin.piped;
|
||||
|
||||
import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress;
|
||||
import static io.activej.http.HttpHeaders.AUTHORIZATION;
|
||||
import static io.activej.http.HttpHeaders.CACHE_CONTROL;
|
||||
import static io.activej.http.HttpHeaders.CONTENT_TYPE;
|
||||
import static io.activej.http.HttpHeaders.LINK;
|
||||
import static io.activej.http.HttpHeaders.LOCATION;
|
||||
import static io.activej.http.HttpMethod.GET;
|
||||
import static io.activej.http.HttpMethod.POST;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.hibernate.Session;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.rometools.rome.feed.synd.SyndFeed;
|
||||
import com.rometools.rome.io.SyndFeedInput;
|
||||
|
||||
import io.activej.config.Config;
|
||||
import io.activej.http.AsyncServlet;
|
||||
import io.activej.http.HttpMethod;
|
||||
|
|
@ -33,15 +13,25 @@ import io.activej.inject.module.AbstractModule;
|
|||
import io.activej.inject.module.Module;
|
||||
import io.activej.launchers.http.MultithreadedHttpServerLauncher;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.CustomServletDecorator;
|
||||
import me.kavin.piped.utils.DatabaseSessionFactory;
|
||||
import me.kavin.piped.utils.ExceptionHandler;
|
||||
import me.kavin.piped.utils.Multithreading;
|
||||
import me.kavin.piped.utils.ResponseHelper;
|
||||
import me.kavin.piped.utils.SponsorBlockUtils;
|
||||
import me.kavin.piped.utils.*;
|
||||
import me.kavin.piped.utils.resp.ErrorResponse;
|
||||
import me.kavin.piped.utils.resp.LoginRequest;
|
||||
import me.kavin.piped.utils.resp.SubscriptionUpdateRequest;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.hibernate.Session;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress;
|
||||
import static io.activej.http.HttpHeaders.*;
|
||||
import static io.activej.http.HttpMethod.GET;
|
||||
import static io.activej.http.HttpMethod.POST;
|
||||
|
||||
public class ServerLauncher extends MultithreadedHttpServerLauncher {
|
||||
|
||||
|
|
@ -55,9 +45,8 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
|
|||
|
||||
RoutingServlet router = RoutingServlet.create()
|
||||
.map(HttpMethod.OPTIONS, "/*", request -> HttpResponse.ofCode(200))
|
||||
.map(GET, "/webhooks/pubsub", request -> {
|
||||
return HttpResponse.ok200().withPlainText(request.getQueryParameter("hub.challenge"));
|
||||
}).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
|
||||
.map(GET, "/webhooks/pubsub", request -> HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge"))))
|
||||
.map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
|
||||
try {
|
||||
|
||||
SyndFeed feed = new SyndFeedInput().build(
|
||||
|
|
@ -320,7 +309,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
|
|||
}
|
||||
|
||||
private @NotNull HttpResponse getRawResponse(int code, byte[] body, String contentType, String cache,
|
||||
boolean prefetchProxy) {
|
||||
boolean prefetchProxy) {
|
||||
HttpResponse response = HttpResponse.ofCode(code).withBody(body).withHeader(CONTENT_TYPE, contentType)
|
||||
.withHeader(CACHE_CONTROL, cache);
|
||||
if (prefetchProxy)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ package me.kavin.piped.consts;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
import me.kavin.piped.utils.PageMixin;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.brotli.BrotliInterceptor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
|
|
@ -11,10 +13,6 @@ import org.schabi.newpipe.extractor.StreamingService;
|
|||
import java.io.FileReader;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpClient.Builder;
|
||||
import java.net.http.HttpClient.Redirect;
|
||||
import java.net.http.HttpClient.Version;
|
||||
import java.util.Properties;
|
||||
|
||||
public class Constants {
|
||||
|
|
@ -36,8 +34,8 @@ public class Constants {
|
|||
|
||||
public static final String FRONTEND_URL;
|
||||
|
||||
public static final HttpClient h2client;
|
||||
public static final HttpClient h2_no_redir_client;
|
||||
public static final OkHttpClient h2client;
|
||||
public static final OkHttpClient h2_no_redir_client;
|
||||
|
||||
public static final boolean COMPROMISED_PASSWORD_CHECK;
|
||||
|
||||
|
|
@ -76,8 +74,12 @@ public class Constants {
|
|||
if (key.startsWith("hibernate"))
|
||||
hibernateProperties.put(key, value);
|
||||
});
|
||||
Builder builder = HttpClient.newBuilder().followRedirects(Redirect.NORMAL).version(Version.HTTP_1_1);
|
||||
Builder builder_noredir = HttpClient.newBuilder().followRedirects(Redirect.NEVER).version(Version.HTTP_1_1);
|
||||
OkHttpClient.Builder builder = new OkHttpClient.Builder()
|
||||
.followRedirects(true)
|
||||
.addInterceptor(BrotliInterceptor.INSTANCE);
|
||||
OkHttpClient.Builder builder_noredir = new OkHttpClient.Builder()
|
||||
.followRedirects(false)
|
||||
.addInterceptor(BrotliInterceptor.INSTANCE);
|
||||
if (HTTP_PROXY != null && HTTP_PROXY.contains(":")) {
|
||||
String host = StringUtils.substringBefore(HTTP_PROXY, ":");
|
||||
String port = StringUtils.substringAfter(HTTP_PROXY, ":");
|
||||
|
|
|
|||
|
|
@ -1,21 +1,18 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpRequest.BodyPublishers;
|
||||
import java.net.http.HttpRequest.Builder;
|
||||
import java.net.http.HttpResponse.BodyHandlers;
|
||||
import java.util.Map;
|
||||
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.obj.SolvedCaptcha;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.RequestBody;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
public class CaptchaSolver {
|
||||
|
||||
|
|
@ -29,9 +26,8 @@ public class CaptchaSolver {
|
|||
}
|
||||
|
||||
private static int createTask(String url, String sitekey, String data_s)
|
||||
throws JsonParserException, IOException, InterruptedException {
|
||||
throws JsonParserException, IOException {
|
||||
|
||||
Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "createTask"));
|
||||
JsonObject jObject = new JsonObject();
|
||||
jObject.put("clientKey", Constants.CAPTCHA_API_KEY);
|
||||
{
|
||||
|
|
@ -43,17 +39,16 @@ public class CaptchaSolver {
|
|||
jObject.put("task", task);
|
||||
}
|
||||
|
||||
builder.method("POST", BodyPublishers.ofString(JsonWriter.string(jObject)));
|
||||
|
||||
builder.header("Content-Type", "application/json");
|
||||
var builder = new Request.Builder().url(Constants.CAPTCHA_BASE_URL + "createTask")
|
||||
.post(RequestBody.create(JsonWriter.string(jObject), MediaType.get("application/json")));
|
||||
|
||||
JsonObject taskObj = JsonParser.object()
|
||||
.from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
|
||||
.from(Constants.h2client.newCall(builder.build()).execute().body().byteStream());
|
||||
|
||||
return taskObj.getInt("taskId");
|
||||
}
|
||||
|
||||
private static final SolvedCaptcha waitForSolve(int taskId)
|
||||
private static SolvedCaptcha waitForSolve(int taskId)
|
||||
throws JsonParserException, IOException, InterruptedException {
|
||||
|
||||
String body = JsonWriter.string(
|
||||
|
|
@ -61,15 +56,15 @@ public class CaptchaSolver {
|
|||
|
||||
SolvedCaptcha solved = null;
|
||||
|
||||
outer: while (true) {
|
||||
Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "getTaskResult"));
|
||||
|
||||
builder.method("POST", BodyPublishers.ofString(body));
|
||||
while (true) {
|
||||
var builder = new Request.Builder()
|
||||
.url(Constants.CAPTCHA_BASE_URL + "getTaskResult")
|
||||
.post(RequestBody.create(body, MediaType.get("application/json")));
|
||||
|
||||
builder.header("Content-Type", "application/json");
|
||||
|
||||
JsonObject captchaObj = JsonParser.object()
|
||||
.from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
|
||||
.from(Constants.h2client.newCall(builder.build()).execute().body().byteStream());
|
||||
|
||||
if (captchaObj.getInt("errorId") != 0)
|
||||
break;
|
||||
|
|
@ -87,7 +82,7 @@ public class CaptchaSolver {
|
|||
});
|
||||
|
||||
solved = new SolvedCaptcha(cookies, captchaResp);
|
||||
break outer;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,12 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import io.activej.http.*;
|
||||
import io.activej.promise.Promisable;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS;
|
||||
import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import io.activej.http.AsyncServlet;
|
||||
import io.activej.http.HttpHeader;
|
||||
import io.activej.http.HttpHeaderValue;
|
||||
import io.activej.http.HttpHeaders;
|
||||
import io.activej.http.HttpRequest;
|
||||
import io.activej.http.HttpResponse;
|
||||
import io.activej.promise.Promisable;
|
||||
|
||||
public class CustomServletDecorator implements AsyncServlet {
|
||||
|
||||
private static final HttpHeader HEADER = HttpHeaders.of("Server-Timing");
|
||||
|
|
|
|||
|
|
@ -1,18 +1,13 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.HttpCookie;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpRequest.BodyPublisher;
|
||||
import java.net.http.HttpRequest.BodyPublishers;
|
||||
import java.net.http.HttpRequest.Builder;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.net.http.HttpResponse.BodyHandlers;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import com.github.benmanes.caffeine.cache.Scheduler;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.obj.SolvedCaptcha;
|
||||
import okhttp3.FormBody;
|
||||
import okhttp3.RequestBody;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jsoup.Jsoup;
|
||||
|
|
@ -22,13 +17,9 @@ import org.schabi.newpipe.extractor.downloader.Request;
|
|||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.obj.SolvedCaptcha;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpCookie;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class DownloaderImpl extends Downloader {
|
||||
|
||||
|
|
@ -52,30 +43,24 @@ public class DownloaderImpl extends Downloader {
|
|||
public Response executeRequest(Request request) throws IOException, ReCaptchaException {
|
||||
|
||||
// TODO: HTTP/3 aka QUIC
|
||||
Builder builder = HttpRequest.newBuilder(URI.create(request.url()));
|
||||
var bytes = request.dataToSend();
|
||||
RequestBody body = null;
|
||||
if (bytes != null)
|
||||
body = RequestBody.create(bytes);
|
||||
|
||||
byte[] data = request.dataToSend();
|
||||
BodyPublisher publisher = data == null ? HttpRequest.BodyPublishers.noBody()
|
||||
: HttpRequest.BodyPublishers.ofByteArray(data);
|
||||
|
||||
builder.method(request.httpMethod(), publisher);
|
||||
|
||||
builder.setHeader("User-Agent", Constants.USER_AGENT);
|
||||
var builder = new okhttp3.Request.Builder()
|
||||
.url(request.url())
|
||||
.method(request.httpMethod(), body)
|
||||
.header("User-Agent", Constants.USER_AGENT);
|
||||
|
||||
if (saved_cookie != null && !saved_cookie.hasExpired())
|
||||
builder.setHeader("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
|
||||
builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
|
||||
|
||||
request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
|
||||
|
||||
HttpResponse<String> response = null;
|
||||
var response = Constants.h2client.newCall(builder.build()).execute();
|
||||
|
||||
try {
|
||||
response = Constants.h2client.send(builder.build(), BodyHandlers.ofString());
|
||||
} catch (InterruptedException e) {
|
||||
// ignored
|
||||
}
|
||||
|
||||
if (response.statusCode() == 429) {
|
||||
if (response.code() == 429) {
|
||||
|
||||
synchronized (cookie_lock) {
|
||||
|
||||
|
|
@ -83,25 +68,25 @@ public class DownloaderImpl extends Downloader {
|
|||
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
|
||||
saved_cookie = null;
|
||||
|
||||
String redir_url = String.valueOf(response.request().uri());
|
||||
String redir_url = String.valueOf(response.request().url());
|
||||
|
||||
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
|
||||
|
||||
Map<String, String> formParams = new Object2ObjectOpenHashMap<>();
|
||||
var formBuilder = new FormBody.Builder();
|
||||
String sitekey = null, data_s = null;
|
||||
|
||||
for (Element el : Jsoup.parse(response.body()).selectFirst("form").children()) {
|
||||
for (Element el : Jsoup.parse(response.body().string()).selectFirst("form").children()) {
|
||||
String name;
|
||||
if (!(name = el.tagName()).equals("script")) {
|
||||
if (name.equals("input"))
|
||||
formParams.put(el.attr("name"), el.attr("value"));
|
||||
formBuilder.add(el.attr("name"), el.attr("value"));
|
||||
else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
|
||||
sitekey = el.attr("data-sitekey");
|
||||
data_s = el.attr("data-s");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sitekey == null || data_s == null)
|
||||
if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
|
||||
throw new ReCaptchaException("Could not get recaptcha", redir_url);
|
||||
|
||||
SolvedCaptcha solved = null;
|
||||
|
|
@ -112,35 +97,19 @@ public class DownloaderImpl extends Downloader {
|
|||
e.printStackTrace();
|
||||
}
|
||||
|
||||
formParams.put("g-recaptcha-response", solved.getRecaptchaResponse());
|
||||
formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
|
||||
|
||||
Builder formBuilder = HttpRequest.newBuilder(URI.create("https://www.google.com/sorry/index"));
|
||||
var formReqBuilder = new okhttp3.Request.Builder()
|
||||
.url("https://www.google.com/sorry/index")
|
||||
.header("User-Agent", Constants.USER_AGENT)
|
||||
.post(formBuilder.build());
|
||||
|
||||
formBuilder.setHeader("User-Agent", Constants.USER_AGENT);
|
||||
var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
|
||||
|
||||
StringBuilder formBody = new StringBuilder();
|
||||
|
||||
formParams.forEach((name, value) -> {
|
||||
formBody.append(name + "=" + URLUtils.silentEncode(value) + "&");
|
||||
});
|
||||
|
||||
formBuilder.header("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
formBuilder.method("POST",
|
||||
BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1))));
|
||||
|
||||
try {
|
||||
HttpResponse<String> formResponse = Constants.h2_no_redir_client.send(formBuilder.build(),
|
||||
BodyHandlers.ofString());
|
||||
|
||||
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
|
||||
.substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse=")))
|
||||
.get(0);
|
||||
cookie_received = System.currentTimeMillis();
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
|
||||
.substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
|
||||
.get(0);
|
||||
cookie_received = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
|
||||
|
|
@ -149,7 +118,7 @@ public class DownloaderImpl extends Downloader {
|
|||
|
||||
}
|
||||
|
||||
return new Response(response.statusCode(), "UNDEFINED", response.headers().map(), response.body(),
|
||||
String.valueOf(response.uri()));
|
||||
return new Response(response.code(), response.message(), response.headers().toMultimap(), response.body().string(),
|
||||
String.valueOf(response.request().url()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
|
||||
|
||||
public class ExceptionHandler {
|
||||
|
||||
public static Exception handle(Exception e) {
|
||||
|
|
@ -18,8 +16,7 @@ public class ExceptionHandler {
|
|||
if (e.getCause() != null && (e instanceof ExecutionException || e instanceof CompletionException))
|
||||
e = (Exception) e.getCause();
|
||||
|
||||
if (!(e instanceof AgeRestrictedContentException || e instanceof ContentNotAvailableException
|
||||
|| e instanceof GeographicRestrictionException)) {
|
||||
if (!(e instanceof ContentNotAvailableException)) {
|
||||
if (path != null)
|
||||
System.err.println("An error occoured in the path: " + path);
|
||||
e.printStackTrace();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.RequestBody;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class LbryHelper {
|
||||
|
||||
public static String getLBRYId(String videoId) throws IOException, InterruptedException {
|
||||
return new JSONObject(
|
||||
RequestUtils.sendGet("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId))
|
||||
).getJSONObject("data").getJSONObject("videos").optString(videoId, null);
|
||||
}
|
||||
|
||||
public static String getLBRYStreamURL(String lbryId)
|
||||
throws IOException {
|
||||
var request = new Request.Builder()
|
||||
.url("https://api.lbry.tv/api/v1/proxy?m=get")
|
||||
.post(RequestBody.create(String.valueOf(
|
||||
new JSONObject().put("id", System.currentTimeMillis())
|
||||
.put("jsonrpc", "2.0")
|
||||
.put("method", "get")
|
||||
.put("params",
|
||||
new JSONObject()
|
||||
.put("uri", "lbry://" + lbryId)
|
||||
.put("save_file", true)))
|
||||
, MediaType.get("application/json")))
|
||||
.build();
|
||||
|
||||
return new JSONObject(
|
||||
Constants.h2client.newCall(request).execute().body().string()
|
||||
).getJSONObject("result").getString("streaming_url");
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +1,20 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse.BodyHandlers;
|
||||
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import okhttp3.Request;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class RequestUtils {
|
||||
|
||||
public static String sendGet(String url) throws IOException, InterruptedException, URISyntaxException {
|
||||
public static String sendGet(String url) throws IOException {
|
||||
return sendGet(url, Constants.USER_AGENT);
|
||||
}
|
||||
|
||||
public static String sendGet(String url, String ua) throws IOException, InterruptedException, URISyntaxException {
|
||||
public static String sendGet(String url, String ua) throws IOException {
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder(new URI(url)).GET().setHeader("User-Agent", ua).build();
|
||||
var request = new Request.Builder().header("User-Agent", ua).url(url).build();
|
||||
|
||||
return Constants.h2client.send(request, BodyHandlers.ofString()).body();
|
||||
return Constants.h2client.newCall(request).execute().body().string();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,41 +1,28 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpRequest.BodyPublishers;
|
||||
import java.net.http.HttpRequest.Builder;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.net.http.HttpResponse.BodyHandlers;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.spec.InvalidKeySpecException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.persistence.criteria.CriteriaBuilder;
|
||||
import javax.persistence.criteria.CriteriaQuery;
|
||||
import javax.persistence.criteria.Root;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import com.github.benmanes.caffeine.cache.Scheduler;
|
||||
import com.rometools.rome.feed.synd.*;
|
||||
import com.rometools.rome.io.FeedException;
|
||||
import com.rometools.rome.io.SyndFeedOutput;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.ipfs.IPFS;
|
||||
import me.kavin.piped.utils.obj.*;
|
||||
import me.kavin.piped.utils.obj.db.PubSub;
|
||||
import me.kavin.piped.utils.obj.db.User;
|
||||
import me.kavin.piped.utils.obj.db.Video;
|
||||
import me.kavin.piped.utils.obj.search.SearchChannel;
|
||||
import me.kavin.piped.utils.obj.search.SearchPlaylist;
|
||||
import me.kavin.piped.utils.resp.*;
|
||||
import okhttp3.FormBody;
|
||||
import okhttp3.Request;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.hibernate.Session;
|
||||
import org.json.JSONObject;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
|
||||
import org.schabi.newpipe.extractor.Page;
|
||||
|
|
@ -44,7 +31,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
|
|||
import org.schabi.newpipe.extractor.comments.CommentsInfo;
|
||||
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskList;
|
||||
|
|
@ -58,49 +44,19 @@ import org.schabi.newpipe.extractor.stream.StreamType;
|
|||
import org.springframework.security.crypto.argon2.Argon2PasswordEncoder;
|
||||
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import com.rometools.rome.feed.synd.SyndEntry;
|
||||
import com.rometools.rome.feed.synd.SyndEntryImpl;
|
||||
import com.rometools.rome.feed.synd.SyndFeed;
|
||||
import com.rometools.rome.feed.synd.SyndFeedImpl;
|
||||
import com.rometools.rome.feed.synd.SyndPerson;
|
||||
import com.rometools.rome.feed.synd.SyndPersonImpl;
|
||||
import com.rometools.rome.io.FeedException;
|
||||
import com.rometools.rome.io.SyndFeedOutput;
|
||||
import javax.persistence.criteria.CriteriaBuilder;
|
||||
import javax.persistence.criteria.CriteriaQuery;
|
||||
import javax.persistence.criteria.Root;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.ipfs.IPFS;
|
||||
import me.kavin.piped.utils.obj.Channel;
|
||||
import me.kavin.piped.utils.obj.ChapterSegment;
|
||||
import me.kavin.piped.utils.obj.Comment;
|
||||
import me.kavin.piped.utils.obj.CommentsPage;
|
||||
import me.kavin.piped.utils.obj.FeedItem;
|
||||
import me.kavin.piped.utils.obj.PipedStream;
|
||||
import me.kavin.piped.utils.obj.Playlist;
|
||||
import me.kavin.piped.utils.obj.SearchResults;
|
||||
import me.kavin.piped.utils.obj.StreamItem;
|
||||
import me.kavin.piped.utils.obj.Streams;
|
||||
import me.kavin.piped.utils.obj.StreamsPage;
|
||||
import me.kavin.piped.utils.obj.SubscriptionChannel;
|
||||
import me.kavin.piped.utils.obj.Subtitle;
|
||||
import me.kavin.piped.utils.obj.db.PubSub;
|
||||
import me.kavin.piped.utils.obj.db.User;
|
||||
import me.kavin.piped.utils.obj.db.Video;
|
||||
import me.kavin.piped.utils.obj.search.SearchChannel;
|
||||
import me.kavin.piped.utils.obj.search.SearchPlaylist;
|
||||
import me.kavin.piped.utils.resp.AcceptedResponse;
|
||||
import me.kavin.piped.utils.resp.AlreadyRegisteredResponse;
|
||||
import me.kavin.piped.utils.resp.AuthenticationFailureResponse;
|
||||
import me.kavin.piped.utils.resp.CompromisedPasswordResponse;
|
||||
import me.kavin.piped.utils.resp.DisabledRegistrationResponse;
|
||||
import me.kavin.piped.utils.resp.IncorrectCredentialsResponse;
|
||||
import me.kavin.piped.utils.resp.InvalidRequestResponse;
|
||||
import me.kavin.piped.utils.resp.LoginResponse;
|
||||
import me.kavin.piped.utils.resp.SubscribeStatusResponse;
|
||||
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
||||
import static me.kavin.piped.utils.URLUtils.rewriteURL;
|
||||
import static me.kavin.piped.utils.URLUtils.substringYouTube;
|
||||
|
||||
public class ResponseHelper {
|
||||
|
||||
|
|
@ -122,7 +78,7 @@ public class ResponseHelper {
|
|||
|
||||
CompletableFuture<String> futureLbryId = CompletableFuture.supplyAsync(() -> {
|
||||
try {
|
||||
return getLBRYId(videoId);
|
||||
return LbryHelper.getLBRYId(videoId);
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.handle(e);
|
||||
}
|
||||
|
|
@ -135,7 +91,7 @@ public class ResponseHelper {
|
|||
|
||||
lbryId = futureLbryId.completeOnTimeout(null, 2, TimeUnit.SECONDS).get();
|
||||
|
||||
return getLBRYStreamURL(lbryId);
|
||||
return LbryHelper.getLBRYStreamURL(lbryId);
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.handle(e);
|
||||
}
|
||||
|
|
@ -194,15 +150,10 @@ public class ResponseHelper {
|
|||
|
||||
info.getRelatedItems().forEach(o -> relatedStreams.add(collectRelatedStream(o)));
|
||||
|
||||
List<ChapterSegment> segments = new ObjectArrayList<>();
|
||||
|
||||
info.getStreamSegments().forEach(segment -> segments
|
||||
.add(new ChapterSegment(segment.getTitle(), segment.getPreviewUrl(), segment.getStartTimeSeconds())));
|
||||
|
||||
long time = info.getUploadDate() != null ? info.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||
: System.currentTimeMillis();
|
||||
|
||||
if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10))
|
||||
if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
||||
updateViews(info.getId(), info.getViewCount(), time, false);
|
||||
|
||||
final Streams streams = new Streams(info.getName(), info.getDescription().getContent(),
|
||||
|
|
@ -243,7 +194,7 @@ public class ResponseHelper {
|
|||
long time = item.getUploadDate() != null
|
||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||
: System.currentTimeMillis();
|
||||
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10))
|
||||
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
||||
updateViews(item.getUrl().substring("https://www.youtube.com/watch?v=".length()),
|
||||
item.getViewCount(), time, true);
|
||||
}
|
||||
|
|
@ -269,7 +220,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] channelPageResponse(String channelId, String prevpageStr)
|
||||
throws IOException, ExtractionException, InterruptedException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
|
||||
|
||||
|
|
@ -293,7 +244,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] trendingResponse(String region)
|
||||
throws ParsingException, ExtractionException, IOException {
|
||||
throws ExtractionException, IOException {
|
||||
|
||||
if (region == null)
|
||||
return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse());
|
||||
|
|
@ -312,7 +263,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] playlistResponse(String playlistId)
|
||||
throws IOException, ExtractionException, InterruptedException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
|
||||
|
||||
|
|
@ -337,7 +288,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] playlistPageResponse(String playlistId, String prevpageStr)
|
||||
throws IOException, ExtractionException, InterruptedException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
|
||||
|
||||
|
|
@ -361,7 +312,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] playlistRSSResponse(String playlistId)
|
||||
throws IOException, ExtractionException, InterruptedException, FeedException {
|
||||
throws IOException, ExtractionException, FeedException {
|
||||
|
||||
final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
|
||||
|
||||
|
|
@ -392,22 +343,24 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] suggestionsResponse(String query)
|
||||
throws JsonProcessingException, IOException, ExtractionException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
return Constants.mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query));
|
||||
|
||||
}
|
||||
|
||||
public static final byte[] opensearchSuggestionsResponse(String query)
|
||||
throws JsonProcessingException, IOException, ExtractionException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
return Constants.mapper.writeValueAsBytes(
|
||||
Arrays.asList(query, YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)));
|
||||
return Constants.mapper.writeValueAsBytes(Arrays.asList(
|
||||
query,
|
||||
YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)
|
||||
));
|
||||
|
||||
}
|
||||
|
||||
public static final byte[] searchResponse(String q, String filter)
|
||||
throws IOException, ExtractionException, InterruptedException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE,
|
||||
YOUTUBE_SERVICE.getSearchQHFactory().fromQuery(q, Collections.singletonList(filter), null));
|
||||
|
|
@ -443,7 +396,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] searchPageResponse(String q, String filter, String prevpageStr)
|
||||
throws IOException, ExtractionException, InterruptedException {
|
||||
throws IOException, ExtractionException {
|
||||
|
||||
Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
|
||||
|
||||
|
|
@ -552,8 +505,8 @@ public class ResponseHelper {
|
|||
|
||||
private static final Argon2PasswordEncoder argon2PasswordEncoder = new Argon2PasswordEncoder();
|
||||
|
||||
public static final byte[] registerResponse(String user, String pass) throws IOException, NoSuchAlgorithmException,
|
||||
InvalidKeySpecException, InterruptedException, URISyntaxException {
|
||||
public static final byte[] registerResponse(String user, String pass) throws IOException,
|
||||
InterruptedException, URISyntaxException {
|
||||
|
||||
if (Constants.DISABLE_REGISTRATION)
|
||||
return Constants.mapper.writeValueAsBytes(new DisabledRegistrationResponse());
|
||||
|
|
@ -602,7 +555,7 @@ public class ResponseHelper {
|
|||
private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
|
||||
|
||||
public static final byte[] loginResponse(String user, String pass)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
if (user == null || pass == null)
|
||||
return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse());
|
||||
|
|
@ -637,7 +590,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] subscribeResponse(String session, String channelId)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -684,7 +637,7 @@ public class ResponseHelper {
|
|||
long time = item.getUploadDate() != null
|
||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||
: System.currentTimeMillis();
|
||||
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10))
|
||||
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
||||
handleNewVideo(item.getUrl(), time, channel, sess);
|
||||
}
|
||||
}
|
||||
|
|
@ -703,7 +656,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] unsubscribeResponse(String session, String channelId)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -725,7 +678,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] isSubscribedResponse(String session, String channelId)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -747,7 +700,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] feedResponse(String session)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -774,7 +727,7 @@ public class ResponseHelper {
|
|||
|
||||
});
|
||||
|
||||
Collections.sort(feedItems, (a, b) -> (int) (b.uploaded - a.uploaded));
|
||||
feedItems.sort((a, b) -> (int) (b.uploaded - a.uploaded));
|
||||
|
||||
s.close();
|
||||
|
||||
|
|
@ -788,7 +741,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] feedResponseRSS(String session)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException, FeedException {
|
||||
throws IOException, FeedException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -811,8 +764,7 @@ public class ResponseHelper {
|
|||
.setParameter("user", user.getId()).addEntity("Video", Video.class)
|
||||
.addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList();
|
||||
|
||||
Collections.sort(queryResults,
|
||||
(a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded()));
|
||||
queryResults.sort((a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded()));
|
||||
|
||||
final List<SyndEntry> entries = new ObjectArrayList<>();
|
||||
|
||||
|
|
@ -850,7 +802,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] importResponse(String session, String[] channelIds, boolean override)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -884,7 +836,7 @@ public class ResponseHelper {
|
|||
me.kavin.piped.utils.obj.db.Channel channel = DatabaseHelper.getChannelFromId(sess, channelId);
|
||||
|
||||
if (channel == null) {
|
||||
ChannelInfo info = null;
|
||||
ChannelInfo info;
|
||||
|
||||
try {
|
||||
info = ChannelInfo.getInfo("https://youtube.com/channel/" + channelId);
|
||||
|
|
@ -910,7 +862,7 @@ public class ResponseHelper {
|
|||
long time = item.getUploadDate() != null
|
||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||
: System.currentTimeMillis();
|
||||
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10))
|
||||
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
||||
handleNewVideo(item.getUrl(), time, channel, sess);
|
||||
}
|
||||
|
||||
|
|
@ -939,7 +891,7 @@ public class ResponseHelper {
|
|||
}
|
||||
|
||||
public static final byte[] subscriptionsResponse(String session)
|
||||
throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
|
||||
throws IOException {
|
||||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
|
|
@ -954,12 +906,10 @@ public class ResponseHelper {
|
|||
List<me.kavin.piped.utils.obj.db.Channel> channels = DatabaseHelper.getChannelFromIds(s,
|
||||
user.getSubscribed());
|
||||
|
||||
channels.forEach(channel -> {
|
||||
subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(),
|
||||
channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified()));
|
||||
});
|
||||
channels.forEach(channel -> subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(),
|
||||
channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified())));
|
||||
|
||||
Collections.sort(subscriptionItems, (a, b) -> (a.name.compareTo(b.name)));
|
||||
subscriptionItems.sort(Comparator.comparing(o -> o.name));
|
||||
}
|
||||
|
||||
s.close();
|
||||
|
|
@ -977,37 +927,11 @@ public class ResponseHelper {
|
|||
|
||||
Session s = DatabaseSessionFactory.createSession();
|
||||
|
||||
long registered = ((Long) s.createQuery("select count(*) from User").uniqueResult()).longValue();
|
||||
long registered = (Long) s.createQuery("select count(*) from User").uniqueResult();
|
||||
|
||||
s.close();
|
||||
|
||||
return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", String.valueOf(registered));
|
||||
}
|
||||
|
||||
private static final String getLBRYId(String videoId) throws IOException, InterruptedException {
|
||||
return new JSONObject(Constants.h2client.send(HttpRequest
|
||||
.newBuilder(URI.create("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId)))
|
||||
.setHeader("User-Agent", Constants.USER_AGENT).build(), BodyHandlers.ofString()).body())
|
||||
.getJSONObject("data").getJSONObject("videos").optString(videoId);
|
||||
}
|
||||
|
||||
private static final String getLBRYStreamURL(String lbryId)
|
||||
throws IOException, InterruptedException, ExecutionException {
|
||||
|
||||
if (lbryId != null && !lbryId.isEmpty())
|
||||
return new JSONObject(
|
||||
Constants.h2client.send(
|
||||
HttpRequest.newBuilder(URI.create("https://api.lbry.tv/api/v1/proxy?m=get"))
|
||||
.POST(BodyPublishers.ofString(
|
||||
String.valueOf(new JSONObject().put("id", System.currentTimeMillis())
|
||||
.put("jsonrpc", "2.0").put("method", "get").put("params",
|
||||
new JSONObject().put("uri", "lbry://" + lbryId)
|
||||
.put("save_file", true)))))
|
||||
.build(),
|
||||
BodyHandlers.ofString()).body()).getJSONObject("result").getString("streaming_url");
|
||||
|
||||
return null;
|
||||
|
||||
return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", registered);
|
||||
}
|
||||
|
||||
public static void handleNewVideo(String url, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) {
|
||||
|
|
@ -1018,8 +942,7 @@ public class ResponseHelper {
|
|||
}
|
||||
}
|
||||
|
||||
private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel,
|
||||
Session s) {
|
||||
private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) {
|
||||
|
||||
if (channel == null)
|
||||
channel = DatabaseHelper.getChannelFromId(s,
|
||||
|
|
@ -1031,7 +954,7 @@ public class ResponseHelper {
|
|||
Video video = null;
|
||||
|
||||
if (channel != null && (video = DatabaseHelper.getVideoFromId(s, info.getId())) == null
|
||||
&& (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(10)) {
|
||||
&& (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) {
|
||||
|
||||
video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(),
|
||||
Math.max(infoTime, time), info.getThumbnailUrl(), channel);
|
||||
|
|
@ -1084,29 +1007,22 @@ public class ResponseHelper {
|
|||
String callback = Constants.PUBLIC_URL + "/webhooks/pubsub";
|
||||
String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId;
|
||||
|
||||
Builder builder = HttpRequest.newBuilder(URI.create("https://pubsubhubbub.appspot.com/subscribe"));
|
||||
var builder = new Request.Builder()
|
||||
.url("https://pubsubhubbub.appspot.com/subscribe");
|
||||
|
||||
Map<String, String> formParams = new Object2ObjectOpenHashMap<>();
|
||||
StringBuilder formBody = new StringBuilder();
|
||||
var formBuilder = new FormBody.Builder();
|
||||
|
||||
builder.header("content-type", "application/x-www-form-urlencoded");
|
||||
formBuilder.add("hub.callback", callback);
|
||||
formBuilder.add("hub.topic", topic);
|
||||
formBuilder.add("hub.verify", "async");
|
||||
formBuilder.add("hub.mode", "subscribe");
|
||||
formBuilder.add("hub.lease_seconds", "432000");
|
||||
|
||||
formParams.put("hub.callback", callback);
|
||||
formParams.put("hub.topic", topic);
|
||||
formParams.put("hub.verify", "async");
|
||||
formParams.put("hub.mode", "subscribe");
|
||||
formParams.put("hub.lease_seconds", "432000");
|
||||
var resp = Constants.h2client
|
||||
.newCall(builder.post(formBuilder.build())
|
||||
.build()).execute();
|
||||
|
||||
formParams.forEach((name, value) -> {
|
||||
formBody.append(name + "=" + URLUtils.silentEncode(value) + "&");
|
||||
});
|
||||
|
||||
builder.method("POST",
|
||||
BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1))));
|
||||
|
||||
HttpResponse<InputStream> resp = Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream());
|
||||
|
||||
if (resp.statusCode() == 202) {
|
||||
if (resp.code() == 202) {
|
||||
if (pubsub == null)
|
||||
pubsub = new PubSub(channelId, System.currentTimeMillis());
|
||||
else
|
||||
|
|
@ -1118,16 +1034,11 @@ public class ResponseHelper {
|
|||
s.getTransaction().begin();
|
||||
s.getTransaction().commit();
|
||||
} else
|
||||
System.out.println(
|
||||
"Failed to subscribe: " + resp.statusCode() + "\n" + IOUtils.toString(resp.body(), "UTF-8"));
|
||||
System.out.println("Failed to subscribe: " + resp.code() + "\n" + resp.body().string());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static final String substringYouTube(String s) {
|
||||
return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com");
|
||||
}
|
||||
|
||||
private static StreamItem collectRelatedStream(Object o) {
|
||||
|
||||
StreamInfoItem item = (StreamInfoItem) o;
|
||||
|
|
@ -1137,35 +1048,4 @@ public class ResponseHelper {
|
|||
rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(), item.getDuration(),
|
||||
item.getViewCount(), item.isUploaderVerified());
|
||||
}
|
||||
|
||||
private static String rewriteURL(final String old) {
|
||||
|
||||
if (old == null || old.isEmpty())
|
||||
return null;
|
||||
|
||||
URL url = null;
|
||||
|
||||
try {
|
||||
url = new URL(old);
|
||||
} catch (MalformedURLException e) {
|
||||
ExceptionHandler.handle(e);
|
||||
}
|
||||
|
||||
final String host = url.getHost();
|
||||
|
||||
String query = url.getQuery();
|
||||
|
||||
boolean hasQuery = query != null;
|
||||
|
||||
String path = url.getPath();
|
||||
|
||||
if (path.contains("=")) {
|
||||
path = StringUtils.substringBefore(path, "=") + "="
|
||||
+ StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
|
||||
}
|
||||
|
||||
return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=")
|
||||
+ URLUtils.silentEncode(host);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,46 +1,36 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import com.grack.nanojson.*;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.resp.InvalidRequestResponse;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse.BodyHandlers;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.resp.InvalidRequestResponse;
|
||||
|
||||
public class SponsorBlockUtils {
|
||||
|
||||
public static final String getSponsors(String id, String categories)
|
||||
throws IOException, InterruptedException, NoSuchAlgorithmException, JsonParserException {
|
||||
public static String getSponsors(String id, String categories)
|
||||
throws IOException, NoSuchAlgorithmException, JsonParserException {
|
||||
|
||||
if (StringUtils.isEmpty(categories))
|
||||
return Constants.mapper.writeValueAsString(new InvalidRequestResponse());
|
||||
|
||||
String hash = toSha256(id);
|
||||
|
||||
URI uri = URI.create("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
|
||||
+ "?categories=" + URLUtils.silentEncode(categories));
|
||||
|
||||
JsonArray jArray = JsonParser.array().from(
|
||||
Constants.h2client.send(HttpRequest.newBuilder(uri).build(), BodyHandlers.ofInputStream()).body());
|
||||
RequestUtils.sendGet("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
|
||||
+ "?categories=" + URLUtils.silentEncode(categories))
|
||||
);
|
||||
|
||||
jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id));
|
||||
|
||||
return JsonWriter.string(jArray.getObject(0));
|
||||
}
|
||||
|
||||
private static final String toSha256(final String videoId) throws NoSuchAlgorithmException {
|
||||
private static String toSha256(final String videoId) throws NoSuchAlgorithmException {
|
||||
final MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
final byte[] bytes = digest.digest(videoId.getBytes(StandardCharsets.UTF_8));
|
||||
final StringBuilder sb = new StringBuilder();
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class URLUtils {
|
||||
|
||||
public static String silentEncode(String s) {
|
||||
try {
|
||||
return URLEncoder.encode(s, "UTF-8");
|
||||
return URLEncoder.encode(s, StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
// ignored
|
||||
}
|
||||
|
|
@ -16,10 +22,42 @@ public class URLUtils {
|
|||
|
||||
public static String silentDecode(String s) {
|
||||
try {
|
||||
return URLDecoder.decode(s, "UTF-8");
|
||||
return URLDecoder.decode(s, StandardCharsets.UTF_8);
|
||||
} catch (Exception e) {
|
||||
// ignored
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
public static String substringYouTube(String s) {
|
||||
return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com");
|
||||
}
|
||||
|
||||
public static String rewriteURL(final String old) {
|
||||
|
||||
if (StringUtils.isEmpty(old)) return null;
|
||||
|
||||
URL url = null;
|
||||
try {
|
||||
url = new URL(old);
|
||||
} catch (MalformedURLException e) {
|
||||
ExceptionHandler.handle(e);
|
||||
}
|
||||
assert url != null;
|
||||
|
||||
final String host = url.getHost();
|
||||
|
||||
String query = url.getQuery();
|
||||
|
||||
boolean hasQuery = query != null;
|
||||
|
||||
String path = url.getPath();
|
||||
|
||||
if (path.contains("=")) {
|
||||
path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
|
||||
}
|
||||
|
||||
return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import javax.persistence.Index;
|
|||
import javax.persistence.Table;
|
||||
|
||||
@Entity
|
||||
@Table(name = "channels", indexes = { @Index(columnList = "uploader_id", name = "channels_uploader_id_idx") })
|
||||
@Table(name = "channels", indexes = {@Index(columnList = "uploader_id", name = "channels_uploader_id_idx")})
|
||||
public class Channel {
|
||||
|
||||
@Id
|
||||
|
|
|
|||
|
|
@ -1,13 +1,6 @@
|
|||
package me.kavin.piped.utils.obj.db;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.FetchType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Index;
|
||||
import javax.persistence.JoinColumn;
|
||||
import javax.persistence.ManyToOne;
|
||||
import javax.persistence.Table;
|
||||
import javax.persistence.*;
|
||||
|
||||
@Entity
|
||||
@Table(name = "videos", indexes = { @Index(columnList = "id", name = "videos_id_idx"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue