2018-07-13 16:02:40 +00:00
|
|
|
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
|
2017-03-01 17:47:52 +00:00
|
|
|
|
|
|
|
|
2020-02-24 18:03:54 +00:00
|
|
|
import com.grack.nanojson.JsonArray;
|
2020-02-22 22:51:02 +00:00
|
|
|
import com.grack.nanojson.JsonObject;
|
|
|
|
import com.grack.nanojson.JsonParser;
|
|
|
|
import com.grack.nanojson.JsonParserException;
|
2019-10-29 05:00:29 +00:00
|
|
|
import org.jsoup.Jsoup;
|
|
|
|
import org.jsoup.nodes.Document;
|
2019-04-28 20:03:16 +00:00
|
|
|
import org.schabi.newpipe.extractor.downloader.Response;
|
2017-03-01 17:47:52 +00:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
2019-10-29 05:00:29 +00:00
|
|
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
2020-02-22 22:51:02 +00:00
|
|
|
import org.schabi.newpipe.extractor.utils.Parser;
|
2017-03-01 17:47:52 +00:00
|
|
|
|
2019-01-13 11:52:07 +00:00
|
|
|
import java.net.URL;
|
2019-04-28 20:03:16 +00:00
|
|
|
import java.text.ParseException;
|
|
|
|
import java.text.SimpleDateFormat;
|
|
|
|
import java.util.Calendar;
|
|
|
|
import java.util.Date;
|
2019-01-13 11:52:07 +00:00
|
|
|
|
2017-06-29 18:12:55 +00:00
|
|
|
/*
|
2017-03-01 17:47:52 +00:00
|
|
|
* Created by Christian Schabesberger on 02.03.16.
|
|
|
|
*
|
|
|
|
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
|
|
|
* YoutubeParsingHelper.java is part of NewPipe.
|
|
|
|
*
|
|
|
|
* NewPipe is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* NewPipe is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
public class YoutubeParsingHelper {
|
|
|
|
|
|
|
|
private YoutubeParsingHelper() {
|
|
|
|
}
|
|
|
|
|
2020-02-24 18:03:54 +00:00
|
|
|
public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
|
|
|
|
|
2019-12-16 07:35:44 +00:00
|
|
|
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
|
|
|
|
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
|
|
|
|
|
2019-10-29 05:00:29 +00:00
|
|
|
private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
|
|
|
|
"form[action*=\"/das_captcha\"]",
|
|
|
|
"input[name*=\"action_recaptcha_verify\"]"
|
|
|
|
};
|
|
|
|
|
2019-04-28 20:03:16 +00:00
|
|
|
public static Document parseAndCheckPage(final String url, final Response response) throws ReCaptchaException {
|
|
|
|
final Document document = Jsoup.parse(response.responseBody(), url);
|
2019-10-29 05:00:29 +00:00
|
|
|
|
|
|
|
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
|
|
|
|
if (!document.select(detectionSelector).isEmpty()) {
|
|
|
|
throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return document;
|
|
|
|
}
|
|
|
|
|
2019-01-13 11:52:07 +00:00
|
|
|
public static boolean isYoutubeURL(URL url) {
|
|
|
|
String host = url.getHost();
|
|
|
|
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|
2019-09-12 02:43:49 +00:00
|
|
|
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
|
2019-01-13 11:52:07 +00:00
|
|
|
}
|
|
|
|
|
2019-01-27 00:28:51 +00:00
|
|
|
public static boolean isYoutubeServiceURL(URL url) {
|
|
|
|
String host = url.getHost();
|
|
|
|
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
|
|
|
|
}
|
2019-01-13 11:52:07 +00:00
|
|
|
|
2019-01-27 00:28:51 +00:00
|
|
|
public static boolean isHooktubeURL(URL url) {
|
2019-01-13 11:52:07 +00:00
|
|
|
String host = url.getHost();
|
2019-01-27 00:28:51 +00:00
|
|
|
return host.equalsIgnoreCase("hooktube.com");
|
|
|
|
}
|
|
|
|
|
|
|
|
public static boolean isInvidioURL(URL url) {
|
|
|
|
String host = url.getHost();
|
2019-09-10 16:54:32 +00:00
|
|
|
return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org");
|
2019-01-13 11:52:07 +00:00
|
|
|
}
|
|
|
|
|
2017-08-11 18:21:49 +00:00
|
|
|
public static long parseDurationString(String input)
|
2017-03-01 17:47:52 +00:00
|
|
|
throws ParsingException, NumberFormatException {
|
2018-09-09 09:53:10 +00:00
|
|
|
|
|
|
|
// If time separator : is not detected, try . instead
|
2018-09-09 12:01:39 +00:00
|
|
|
|
|
|
|
final String[] splitInput = input.contains(":")
|
|
|
|
? input.split(":")
|
|
|
|
: input.split("\\.");
|
|
|
|
|
2017-03-01 17:47:52 +00:00
|
|
|
String days = "0";
|
|
|
|
String hours = "0";
|
|
|
|
String minutes = "0";
|
2018-09-09 12:01:39 +00:00
|
|
|
final String seconds;
|
2017-03-01 17:47:52 +00:00
|
|
|
|
2017-06-29 18:12:55 +00:00
|
|
|
switch (splitInput.length) {
|
2017-03-01 17:47:52 +00:00
|
|
|
case 4:
|
|
|
|
days = splitInput[0];
|
|
|
|
hours = splitInput[1];
|
|
|
|
minutes = splitInput[2];
|
|
|
|
seconds = splitInput[3];
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
hours = splitInput[0];
|
|
|
|
minutes = splitInput[1];
|
|
|
|
seconds = splitInput[2];
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
minutes = splitInput[0];
|
|
|
|
seconds = splitInput[1];
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
seconds = splitInput[0];
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
throw new ParsingException("Error duration string with unknown format: " + input);
|
|
|
|
}
|
2017-08-11 18:21:49 +00:00
|
|
|
return ((((Long.parseLong(days) * 24)
|
|
|
|
+ Long.parseLong(hours) * 60)
|
|
|
|
+ Long.parseLong(minutes)) * 60)
|
|
|
|
+ Long.parseLong(seconds);
|
2017-03-01 17:47:52 +00:00
|
|
|
}
|
2019-04-28 20:03:16 +00:00
|
|
|
|
2019-12-16 07:35:44 +00:00
|
|
|
public static String getFeedUrlFrom(final String channelIdOrUser) {
|
|
|
|
if (channelIdOrUser.startsWith("user/")) {
|
|
|
|
return FEED_BASE_USER + channelIdOrUser.replace("user/", "");
|
|
|
|
} else if (channelIdOrUser.startsWith("channel/")) {
|
|
|
|
return FEED_BASE_CHANNEL_ID + channelIdOrUser.replace("channel/", "");
|
|
|
|
} else {
|
|
|
|
return FEED_BASE_CHANNEL_ID + channelIdOrUser;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-28 20:03:16 +00:00
|
|
|
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
|
|
|
|
Date date;
|
|
|
|
try {
|
|
|
|
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
|
|
|
|
} catch (ParseException e) {
|
|
|
|
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
|
|
|
|
}
|
|
|
|
|
|
|
|
final Calendar uploadDate = Calendar.getInstance();
|
|
|
|
uploadDate.setTime(date);
|
|
|
|
return uploadDate;
|
|
|
|
}
|
2020-02-22 22:51:02 +00:00
|
|
|
|
|
|
|
public static JsonObject getInitialData(String html) throws ParsingException {
|
|
|
|
try {
|
|
|
|
String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
|
|
|
|
return JsonParser.object().from(initialData);
|
|
|
|
} catch (JsonParserException | Parser.RegexException e) {
|
|
|
|
throw new ParsingException("Could not get ytInitialData", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-24 18:03:54 +00:00
|
|
|
/**
|
|
|
|
* Get the client version from a page
|
|
|
|
* @param initialData
|
|
|
|
* @param html The page HTML
|
|
|
|
* @return
|
|
|
|
* @throws ParsingException
|
|
|
|
*/
|
|
|
|
public static String getClientVersion(JsonObject initialData, String html) throws ParsingException {
|
|
|
|
if (initialData == null) initialData = getInitialData(html);
|
|
|
|
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
|
|
|
|
String shortClientVersion = null;
|
|
|
|
|
|
|
|
// try to get version from initial data first
|
|
|
|
for (Object service : serviceTrackingParams) {
|
|
|
|
JsonObject s = (JsonObject) service;
|
|
|
|
if (s.getString("service").equals("CSI")) {
|
|
|
|
JsonArray params = s.getArray("params");
|
|
|
|
for (Object param: params) {
|
|
|
|
JsonObject p = (JsonObject) param;
|
|
|
|
String key = p.getString("key");
|
|
|
|
if (key != null && key.equals("cver")) {
|
|
|
|
return p.getString("value");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (s.getString("service").equals("ECATCHER")) {
|
|
|
|
// fallback to get a shortened client version which does not contain the last do digits
|
|
|
|
JsonArray params = s.getArray("params");
|
|
|
|
for (Object param: params) {
|
|
|
|
JsonObject p = (JsonObject) param;
|
|
|
|
String key = p.getString("key");
|
|
|
|
if (key != null && key.equals("client.version")) {
|
|
|
|
shortClientVersion = p.getString("value");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
String clientVersion;
|
|
|
|
String[] patterns = {
|
|
|
|
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
|
|
|
|
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
|
|
|
|
"client.version=([0-9\\.]+)"
|
|
|
|
};
|
|
|
|
for (String pattern: patterns) {
|
|
|
|
try {
|
|
|
|
clientVersion = Parser.matchGroup1(pattern, html);
|
|
|
|
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
|
|
|
|
} catch (Exception ignored) {}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (shortClientVersion != null) return shortClientVersion;
|
|
|
|
|
|
|
|
throw new ParsingException("Could not get client version");
|
|
|
|
}
|
2017-03-01 17:47:52 +00:00
|
|
|
}
|