CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Adress reviews and do some improvements

Browse source 
Adress changes requested in reviews.
Do some improvements, remove unused imports and format some code to be in the 100 characters line limit.
This commit is contained in:
TiA4f8R 2021-04-25 18:54:26 +02:00
parent e075dd5a63
commit f46cfb0f26
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
4 changed files with 95 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

149
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file

@ -8,6 +8,7 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter; import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*; import org.schabi.newpipe.extractor.exceptions.*;
@ -65,15 +66,16 @@ public class YoutubeParsingHelper {
private static final String HARDCODED_CLIENT_VERSION = "2.20210420.07.00"; private static final String HARDCODED_CLIENT_VERSION = "2.20210420.07.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static final String YOUTUBEI_V1_URL = "https://youtubei.googleapis.com/youtubei/v1/";
private static String clientVersion; private static String clientVersion;
private static String key; private static String key;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS =
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
private static String[] youtubeMusicKeys; private static String[] youtubeMusicKeys;
private static boolean keyAndVersionExtracted = false; private static boolean keyAndVersionExtracted = false;
private static boolean areHardcodedClientVersionAndKeyValidRan = false; private static Boolean areHardcodedClientVersionAndKeyValidValue = null;
private static boolean areHardcodedClientVersionAndKeyValidValue;
private static Random numberGenerator = new Random(); private static Random numberGenerator = new Random();
@ -88,7 +90,8 @@ public class YoutubeParsingHelper {
*/ */
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE; private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_CHANNEL_ID =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static boolean isGoogleURL(String url) { private static boolean isGoogleURL(String url) {
@ -106,12 +109,14 @@ public class YoutubeParsingHelper {
public static boolean isYoutubeURL(final URL url) { public static boolean isYoutubeURL(final URL url) {
final String host = url.getHost(); final String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com"); || host.equalsIgnoreCase("m.youtube.com")
|| host.equalsIgnoreCase("music.youtube.com");
} }
public static boolean isYoutubeServiceURL(final URL url) { public static boolean isYoutubeServiceURL(final URL url) {
final String host = url.getHost(); final String host = url.getHost();
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be"); return host.equalsIgnoreCase("www.youtube-nocookie.com")
|| host.equalsIgnoreCase("youtu.be");
} }
public static boolean isHooktubeURL(final URL url) { public static boolean isHooktubeURL(final URL url) {
@ -207,14 +212,16 @@ public class YoutubeParsingHelper {
} }
} }
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException { public static OffsetDateTime parseDateFrom(final String textualUploadDate)
throws ParsingException {
try { try {
return OffsetDateTime.parse(textualUploadDate); return OffsetDateTime.parse(textualUploadDate);
} catch (final DateTimeParseException e) { } catch (final DateTimeParseException e) {
try { try {
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC); return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
} catch (final DateTimeParseException e1) { } catch (final DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1); throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"",
e1);
} }
} }
} }
@ -265,23 +272,27 @@ public class YoutubeParsingHelper {
} else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM" } else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
// Channel mix are build with RMCM{channelId}, so videoId can't be determined // Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId); throw new ParsingException("Video id could not be determined from mix id: "
+ playlistId);
} else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD" } else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
return playlistId.substring(2); return playlistId.substring(2);
} else { // not a mix } else { // not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId); throw new ParsingException("Video id could not be determined from mix id: "
+ playlistId);
} }
} }
public static JsonObject getInitialData(final String html) throws ParsingException { public static JsonObject getInitialData(final String html) throws ParsingException {
try { try {
try { try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); final String initialData = Parser.matchGroup1(
"window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData); return JsonParser.object().from(initialData);
} catch (final Parser.RegexException e) { } catch (final Parser.RegexException e) {
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html); final String initialData = Parser.matchGroup1(
"var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData); return JsonParser.object().from(initialData);
} }
} catch (final JsonParserException | Parser.RegexException e) { } catch (final JsonParserException | Parser.RegexException e) {
@ -289,8 +300,11 @@ public class YoutubeParsingHelper {
} }
} }
public static boolean areHardcodedClientVersionAndKeyValid() throws IOException, ExtractionException { public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
if (areHardcodedClientVersionAndKeyValidRan) return areHardcodedClientVersionAndKeyValidValue; ExtractionException {
if (areHardcodedClientVersionAndKeyValidValue != null) {
return areHardcodedClientVersionAndKeyValidValue;
}
// @formatter:off // @formatter:off
final byte[] body = JsonWriter.string() final byte[] body = JsonWriter.string()
.object() .object()
@ -307,23 +321,23 @@ public class YoutubeParsingHelper {
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_CLIENT_VERSION)); headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION));
// This endpoint is fetched by the YouTube website to get the items of its main menu and is // This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB) // pretty lightweight (around 30kB)
final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/guide?key=" final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
+ HARDCODED_KEY, headers, body); + HARDCODED_KEY, headers, body);
final String responseBody = response.responseBody(); final String responseBody = response.responseBody();
final int responseCode = response.responseCode(); final int responseCode = response.responseCode();
areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000 areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000
&& responseCode == 200; // Ensure to have a valid response && responseCode == 200; // Ensure to have a valid response
areHardcodedClientVersionAndKeyValidRan = true;
return areHardcodedClientVersionAndKeyValidValue; return areHardcodedClientVersionAndKeyValidValue;
} }
private static void extractClientVersionAndKey() throws IOException, ExtractionException { private static void extractClientVersionAndKey() throws IOException, ExtractionException {
// Don't extract the client version and the innertube API key if it has been already extracted // Don't extract the client version and the innertube key if it has been already extracted
if (!keyAndVersionExtracted) return; if (!keyAndVersionExtracted) return;
// Don't provide a search term in order to have a smaller response // Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query="; final String url = "https://www.youtube.com/results?search_query=";
@ -348,7 +362,8 @@ public class YoutubeParsingHelper {
} }
} }
} else if (s.getString("service").equals("ECATCHER")) { } else if (s.getString("service").equals("ECATCHER")) {
// Fallback to get a shortened client version which does not contain the last two digits // Fallback to get a shortened client version which does not contain the last two
// digits
final JsonArray params = s.getArray("params"); final JsonArray params = s.getArray("params");
for (final Object param : params) { for (final Object param : params) {
final JsonObject p = (JsonObject) param; final JsonObject p = (JsonObject) param;
@ -397,7 +412,9 @@ public class YoutubeParsingHelper {
*/ */
public static String getClientVersion() throws IOException, ExtractionException { public static String getClientVersion() throws IOException, ExtractionException {
if (!isNullOrEmpty(clientVersion)) return clientVersion; if (!isNullOrEmpty(clientVersion)) return clientVersion;
if (areHardcodedClientVersionAndKeyValid()) return clientVersion = HARDCODED_CLIENT_VERSION; if (areHardcodedClientVersionAndKeyValid()) {
return clientVersion = HARDCODED_CLIENT_VERSION;
}
if (!keyAndVersionExtracted) extractClientVersionAndKey(); if (!keyAndVersionExtracted) extractClientVersionAndKey();
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version"); if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
@ -445,8 +462,9 @@ public class YoutubeParsingHelper {
numberGenerator = random; numberGenerator = random;
} }
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException { public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException,
final String url = "https://youtubei.googleapis.com/youtubei/v1/music/get_search_suggestions?alt=json&key=" ReCaptchaException {
final String url = YOUTUBEI_V1_URL + "music/get_search_suggestions?alt=json&key="
+ HARDCODED_YOUTUBE_MUSIC_KEYS[0]; + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
// @formatter:off // @formatter:off
@ -479,8 +497,10 @@ public class YoutubeParsingHelper {
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1])); headers.put("X-YouTube-Client-Name", Collections.singletonList(
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2])); HARDCODED_YOUTUBE_MUSIC_KEYS[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(
HARDCODED_YOUTUBE_MUSIC_KEYS[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com")); headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com")); headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json")); headers.put("Content-Type", Collections.singletonList("application/json"));
@ -488,13 +508,16 @@ public class YoutubeParsingHelper {
final Response response = getDownloader().post(url, headers, json); final Response response = getDownloader().post(url, headers, json);
final String responseBody = response.responseBody(); final String responseBody = response.responseBody();
final int responseCode = response.responseCode(); final int responseCode = response.responseCode();
// Ensure to have a valid response
return responseBody.length() > 500 && responseCode == 200; // Ensure to have a valid response return responseBody.length() > 500 && responseCode == 200;
} }
public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException { public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException,
Parser.RegexException {
if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys; if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys;
if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS; if (areHardcodedYoutubeMusicKeysValid()) {
return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS;
}
final String url = "https://music.youtube.com/"; final String url = "https://music.youtube.com/";
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
@ -508,26 +531,29 @@ public class YoutubeParsingHelper {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
} }
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html); final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),",
html);
String clientVersion; String clientVersion;
try { try {
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1(
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException e) { } catch (final Parser.RegexException e) {
try { try {
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1(
"INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (final Parser.RegexException ee) { } catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); clientVersion = Parser.matchGroup1(
"innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
} }
} }
return youtubeMusicKeys = new String[]{key, clientName, clientVersion}; return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
} }
@Nullable @Nullable
public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint) throws ParsingException { public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint)
throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) { if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("https://www.youtube.com/redirect?")) { if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
@ -552,7 +578,8 @@ public class YoutubeParsingHelper {
} }
} else if (internUrl.startsWith("http")) { } else if (internUrl.startsWith("http")) {
return internUrl; return internUrl;
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) { } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user")
|| internUrl.startsWith("/watch")) {
return "https://www.youtube.com" + internUrl; return "https://www.youtube.com" + internUrl;
} }
} else if (navigationEndpoint.has("browseEndpoint")) { } else if (navigationEndpoint.has("browseEndpoint")) {
@ -569,10 +596,12 @@ public class YoutubeParsingHelper {
return "https://www.youtube.com" + canonicalBaseUrl; return "https://www.youtube.com" + canonicalBaseUrl;
} }
throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\"" + browseEndpoint + "\")"); throw new ParsingException("canonicalBaseUrl is null and browseId is not a channel (\""
+ browseEndpoint + "\")");
} else if (navigationEndpoint.has("watchEndpoint")) { } else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder(); StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId")); url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint
.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) { if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint") url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId")); .getString("playlistId"));
@ -597,7 +626,8 @@ public class YoutubeParsingHelper {
* @return text in the JSON object or {@code null} * @return text in the JSON object or {@code null}
*/ */
@Nullable @Nullable
public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException { public static String getTextFromObject(final JsonObject textObject, final boolean html)
throws ParsingException {
if (isNullOrEmpty(textObject)) return null; if (isNullOrEmpty(textObject)) return null;
if (textObject.has("simpleText")) return textObject.getString("simpleText"); if (textObject.has("simpleText")) return textObject.getString("simpleText");
@ -608,9 +638,11 @@ public class YoutubeParsingHelper {
for (final Object textPart : textObject.getArray("runs")) { for (final Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text"); String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).has("navigationEndpoint")) { if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint")); String url = getUrlFromNavigationEndpoint(((JsonObject) textPart)
.getObject("navigationEndpoint"));
if (!isNullOrEmpty(url)) { if (!isNullOrEmpty(url)) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>"); textBuilder.append("<a href=\"").append(url).append("\">").append(text)
.append("</a>");
continue; continue;
} }
} }
@ -705,8 +737,8 @@ public class YoutubeParsingHelper {
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers); addClientInfoHeaders(headers);
final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/" final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
+ endpoint + "?key=" + getKey(), headers, body, localization); + getKey(), headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
} }
@ -739,7 +771,7 @@ public class YoutubeParsingHelper {
.object("client") .object("client")
.value("clientName", "1") .value("clientName", "1")
.value("clientVersion", getClientVersion()) .value("clientVersion", getClientVersion())
.value("hl", "en-GB") .value("hl", NewPipe.getPreferredLocalization().getLocalizationCode())
.value("gl", contentCountry) .value("gl", contentCountry)
.end() .end()
.end(); .end();
@ -785,7 +817,7 @@ public class YoutubeParsingHelper {
*/ */
public static void addCookieHeader(final Map<String, List<String>> headers) { public static void addCookieHeader(final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) { if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(generateConsentCookie())); headers.put("Cookie", Collections.singletonList(generateConsentCookie()));
} else { } else {
headers.get("Cookie").add(generateConsentCookie()); headers.get("Cookie").add(generateConsentCookie());
} }
@ -859,10 +891,12 @@ public class YoutubeParsingHelper {
final JsonObject sectionContent = (JsonObject) sectionContentObject; final JsonObject sectionContent = (JsonObject) sectionContentObject;
if (sectionContent.has("infoPanelContentRenderer")) { if (sectionContent.has("infoPanelContentRenderer")) {
metaInfo.add(getInfoPanelContent(sectionContent.getObject("infoPanelContentRenderer"))); metaInfo.add(getInfoPanelContent(sectionContent
.getObject("infoPanelContentRenderer")));
} }
if (sectionContent.has("clarificationRenderer")) { if (sectionContent.has("clarificationRenderer")) {
metaInfo.add(getClarificationRendererContent(sectionContent.getObject("clarificationRenderer") metaInfo.add(getClarificationRendererContent(sectionContent
.getObject("clarificationRenderer")
)); ));
} }
@ -888,7 +922,8 @@ public class YoutubeParsingHelper {
final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint( final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint(
infoPanelContentRenderer.getObject("sourceEndpoint")); infoPanelContentRenderer.getObject("sourceEndpoint"));
try { try {
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(metaInfoLinkUrl)))); metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(
metaInfoLinkUrl))));
} catch (final NullPointerException | MalformedURLException e) { } catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e); throw new ParsingException("Could not get metadata info URL", e);
} }
@ -909,8 +944,10 @@ public class YoutubeParsingHelper {
throws ParsingException { throws ParsingException {
final MetaInfo metaInfo = new MetaInfo(); final MetaInfo metaInfo = new MetaInfo();
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle")); final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text")); .getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer
.getObject("text"));
if (title == null || text == null) { if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content"); throw new ParsingException("Could not extract clarification renderer content");
} }
@ -921,7 +958,8 @@ public class YoutubeParsingHelper {
final JsonObject actionButton = clarificationRenderer.getObject("actionButton") final JsonObject actionButton = clarificationRenderer.getObject("actionButton")
.getObject("buttonRenderer"); .getObject("buttonRenderer");
try { try {
final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton.getObject("command")); final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton
.getObject("command"));
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url)))); metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
} catch (final NullPointerException | MalformedURLException e) { } catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e); throw new ParsingException("Could not get metadata info URL", e);
@ -935,13 +973,16 @@ public class YoutubeParsingHelper {
metaInfo.addUrlText(metaInfoLinkText); metaInfo.addUrlText(metaInfoLinkText);
} }
if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer.has("secondarySource")) { if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer
final String url = getUrlFromNavigationEndpoint(clarificationRenderer.getObject("secondaryEndpoint")); .has("secondarySource")) {
final String url = getUrlFromNavigationEndpoint(clarificationRenderer
.getObject("secondaryEndpoint"));
// ignore Google URLs, because those point to a Google search about "Covid-19" // ignore Google URLs, because those point to a Google search about "Covid-19"
if (url != null && !isGoogleURL(url)) { if (url != null && !isGoogleURL(url)) {
try { try {
metaInfo.addUrl(new URL(url)); metaInfo.addUrl(new URL(url));
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource")); final String description = getTextFromObject(clarificationRenderer
.getObject("secondarySource"));
metaInfo.addUrlText(description == null ? url : description); metaInfo.addUrlText(description == null ? url : description);
} catch (final MalformedURLException e) { } catch (final MalformedURLException e) {
throw new ParsingException("Could not get metadata info secondary URL", e); throw new ParsingException("Could not get metadata info secondary URL", e);

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -11,22 +11,17 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View file

@ -17,10 +17,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter; import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;

1
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file

@ -39,7 +39,6 @@ import java.io.IOException;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;