CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use the youtubei API for YouTube playlists

This commit is contained in:
TiA4f8R 2021-04-08 16:17:59 +02:00
parent c97a19d719
commit 5794eb2350
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
3 changed files with 56 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -95,7 +95,7 @@ public class YoutubeParsingHelper {
final String host = u.getHost();
return host.startsWith("google.") || host.startsWith("m.google.")
|| host.startsWith("www.google.");
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
return false;
}
}
@ -207,10 +207,10 @@ public class YoutubeParsingHelper {
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
} catch (final DateTimeParseException e) {
try {
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
} catch (DateTimeParseException e1) {
} catch (final DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
}
}
@ -277,11 +277,11 @@ public class YoutubeParsingHelper {
try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e) {
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
}
} catch (JsonParserException | Parser.RegexException e) {
} catch (final JsonParserException | Parser.RegexException e) {
throw new ParsingException("Could not get ytInitialData", e);
}
}
@ -342,7 +342,7 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion;
break;
}
} catch (Parser.RegexException ignored) {
} catch (final Parser.RegexException ignored) {
}
}
@ -352,10 +352,10 @@ public class YoutubeParsingHelper {
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) {
} catch (final Parser.RegexException ignored) {
}
}
}
@ -468,7 +468,7 @@ public class YoutubeParsingHelper {
String key;
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
}
@ -477,10 +477,10 @@ public class YoutubeParsingHelper {
String clientVersion;
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException e) {
} catch (final Parser.RegexException e) {
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException ee) {
} catch (final Parser.RegexException ee) {
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
}
}
@ -491,7 +491,7 @@ public class YoutubeParsingHelper {
@Nullable
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
@ -508,7 +508,7 @@ public class YoutubeParsingHelper {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) {
} catch (final UnsupportedEncodingException e) {
return null;
}
return url;
@ -662,6 +662,31 @@ public class YoutubeParsingHelper {
return response;
}
public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("set-cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}
public static JsonObject getJsonPostResponse(final String endpoint,
final byte[] body,
final Localization localization)
throws IOException, ExtractionException {
final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/"
+ endpoint + "?key=" + getKey(), new HashMap<>(), body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
@ -885,7 +910,7 @@ public class YoutubeParsingHelper {
metaInfo.addUrl(new URL(url));
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource"));
metaInfo.addUrlText(description == null ? url : description);
} catch (MalformedURLException e) {
} catch (final MalformedURLException e) {
throw new ParsingException("Could not get metadata info secondary URL", e);
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -147,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get channel name", e);
}
}
@ -159,7 +159,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get avatar", e);
}
}
@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
return fixThumbnailUrl(url);
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get banner", e);
}
}
@ -184,7 +184,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getFeedUrl() throws ParsingException {
try {
return YoutubeParsingHelper.getFeedUrlFrom(getId());
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
@ -207,7 +207,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public String getDescription() throws ParsingException {
try {
return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description");
} catch (Exception e) {
} catch (final Exception e) {
throw new ParsingException("Could not get channel description", e);
}
}
@ -293,7 +293,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done())
.getBytes(UTF_8);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key=" + getKey(),
body);
}
@ -314,7 +314,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
JsonObject continuation = null;
for (Object object : videos) {
for (final Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
@ -344,7 +344,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("tabs");
JsonObject videoTab = null;
for (Object tab : tabs) {
for (final Object tab : tabs) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -29,7 +29,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@ -50,11 +50,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final byte[] body = JsonWriter.string(prepareJsonBuilder()
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // show unavailable videos
.done())
.getBytes(UTF_8);
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = initialAjaxJson.getObject(1).getObject("response");
initialData = getJsonPostResponse("browse", body, getExtractorLocalization());
YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo();
@ -251,9 +253,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.done())
.getBytes(UTF_8);
return new Page(
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
body);
return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key="
+ getKey(), body);
} else {
return null;
}