CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Do some improvements to YoutubeStreamExtractor 

Get the real name of the uploader (for autogenerated channels and music artist channels), like before the migration to the JSON pbj.
Do some other improvements, especially reformatting some code to be in the 100 characters line limit and use final where possible.
This commit is contained in:
TiA4f8R 2021-04-15 18:58:59 +02:00
parent 58ce9b04a1
commit b49ae547a3
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
6 changed files with 295 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -63,7 +63,7 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() { private YoutubeParsingHelper() {
} }
private static final String HARDCODED_CLIENT_VERSION = "2.20210408.08.00"; private static final String HARDCODED_CLIENT_VERSION = "2.20210413.07.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static String clientVersion; private static String clientVersion;
private static String key; private static String key;
@ -729,7 +729,7 @@ public class YoutubeParsingHelper {
return JsonUtils.toJsonArray(getValidJsonResponseBody(response)); return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
} }
public static JsonBuilder<JsonObject> prepareJsonBuilder() public static JsonBuilder<JsonObject> prepareJsonBuilder(final String contentCountry)
throws IOException, ExtractionException { throws IOException, ExtractionException {
// @formatter:off // @formatter:off
return JsonObject.builder() return JsonObject.builder()
@ -737,6 +737,8 @@ public class YoutubeParsingHelper {
.object("client") .object("client")
.value("clientName", "1") .value("clientName", "1")
.value("clientVersion", getClientVersion()) .value("clientVersion", getClientVersion())
.value("hl", "en-GB")
.value("gl", contentCountry)
.end() .end()
.end(); .end();
// @formatter:on // @formatter:on

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -73,20 +73,23 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
*/ */
private String redirectedChannelId; private String redirectedChannelId;
public YoutubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) { public YoutubeChannelExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String channel_path = super.getId(); final String channel_path = super.getId();
final String[] channelInfo = channel_path.split("/"); final String[] channelInfo = channel_path.split("/");
final String contentCountry = getExtractorContentCountry().getCountryCode();
String id = ""; String id = "";
// If the url is an URL which is not a /channel URL, we need to use the // If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we // navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we
// couldn't get information about the channel associated with this URL, if there is one. // couldn't get information about the channel associated with this URL, if there is one.
if (!channelInfo[0].equals("channel")) { if (!channelInfo[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareJsonBuilder() final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("url", "https://www.youtube.com/" + channel_path) .value("url", "https://www.youtube.com/" + channel_path)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
@ -131,7 +134,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
int level = 0; int level = 0;
while (level < 3) { while (level < 3) {
final byte[] body = JsonWriter.string(prepareJsonBuilder() final byte[] body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("browseId", id) .value("browseId", id)
.value("params", "EgZ2aWRlb3M%3D") // equals to videos .value("params", "EgZ2aWRlb3M%3D") // equals to videos
.done()) .done())
@ -336,7 +339,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
@ -361,7 +365,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
} }
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException { private Page getNextPageFrom(final JsonObject continuations) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) { if (isNullOrEmpty(continuations)) {
return null; return null;
} }
@ -370,7 +375,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String continuation = continuationEndpoint.getObject("continuationCommand") final String continuation = continuationEndpoint.getObject("continuationCommand")
.getString("token"); .getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder() final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("continuation", continuation) .value("continuation", continuation)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);

69
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -40,7 +40,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonArray initialAjaxJson;
private JsonObject initialData; private JsonObject initialData;
private JsonObject playlistInfo; private JsonObject playlistInfo;
@ -49,8 +48,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
final byte[] body = JsonWriter.string(prepareJsonBuilder() ExtractionException {
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("browseId", "VL" + getId()) .value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // show unavailable videos .value("params", "wgYCCAA%3D") // show unavailable videos
.done()) .done())
@ -63,15 +64,18 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
private JsonObject getUploaderInfo() throws ParsingException { private JsonObject getUploaderInfo() throws ParsingException {
final JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items"); final JsonArray items = initialData.getObject("sidebar")
.getObject("playlistSidebarRenderer").getArray("items");
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); JsonObject videoOwner = items.getObject(1)
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) { if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer"); return videoOwner.getObject("videoOwnerRenderer");
} }
// we might want to create a loop here instead of using duplicated code // we might want to create a loop here instead of using duplicated code
videoOwner = items.getObject(items.size()).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); videoOwner = items.getObject(items.size())
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) { if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer"); return videoOwner.getObject("videoOwnerRenderer");
} }
@ -80,9 +84,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject getPlaylistInfo() throws ParsingException { private JsonObject getPlaylistInfo() throws ParsingException {
try { try {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items") return initialData.getObject("sidebar").getObject("playlistSidebarRenderer")
.getObject(0).getObject("playlistSidebarPrimaryInfoRenderer"); .getArray("items").getObject(0)
} catch (Exception e) { .getObject("playlistSidebarPrimaryInfoRenderer");
} catch (final Exception e) {
throw new ParsingException("Could not get PlaylistInfo", e); throw new ParsingException("Could not get PlaylistInfo", e);
} }
} }
@ -122,7 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint")); return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader url", e); throw new ParsingException("Could not get playlist uploader url", e);
} }
} }
@ -131,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
try { try {
return getTextFromObject(getUploaderInfo().getObject("title")); return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader name", e); throw new ParsingException("Could not get playlist uploader name", e);
} }
} }
@ -142,7 +147,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e); throw new ParsingException("Could not get playlist uploader avatar", e);
} }
} }
@ -157,7 +162,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
try { try {
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get video count from playlist", e); throw new ParsingException("Could not get video count from playlist", e);
} }
} }
@ -186,18 +191,21 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null; Page nextPage = null;
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer") final JsonArray contents = initialData.getObject("contents")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content") .getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("sectionListRenderer").getArray("contents").getObject(0) .getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getObject("itemSectionRenderer").getArray("contents"); .getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) { if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) { for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) { if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment)); collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) { } else if (((JsonObject) segment).getObject("playlistSegmentRenderer")
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer") .has("videoList")) {
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents")); collectStreamsFrom(collector, ((JsonObject) segment)
.getObject("playlistSegmentRenderer").getObject("videoList")
.getObject("playlistVideoListRenderer").getArray("contents"));
} }
} }
@ -214,7 +222,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
@ -235,7 +244,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
} }
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException { private Page getNextPageFrom(final JsonArray contents) throws IOException,
ExtractionException {
if (isNullOrEmpty(contents)) { if (isNullOrEmpty(contents)) {
return null; return null;
} }
@ -248,7 +258,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("continuationCommand") .getObject("continuationCommand")
.getString("token"); .getString("token");
final byte[] body = JsonWriter.string(prepareJsonBuilder() final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.getCountryCode())
.value("continuation", continuation) .value("continuation", continuation)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
@ -260,12 +271,14 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
} }
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { private void collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object video : videos) { for (final Object video : videos) {
if (((JsonObject) video).has("playlistVideoRenderer")) { if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) { collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video)
.getObject("playlistVideoRenderer"), timeAgoParser) {
@Override @Override
public long getViewCount() { public long getViewCount() {
return -1; return -1;
@ -294,11 +307,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse") return "";
/*final JsonArray thumbnails = initialAjaxJson.getObject(1)
.getObject("playerResponse")
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); .getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);*/
} }
@Override @Override

95
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -50,13 +50,16 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchExtractor extends SearchExtractor { public class YoutubeSearchExtractor extends SearchExtractor {
private JsonObject initialData; private JsonObject initialData;
public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) { public YoutubeSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
final String query = super.getSearchString(); final String query = super.getSearchString();
final String contentCountry = getExtractorContentCountry().getCountryCode();
// Get the search parameter of the request // Get the search parameter of the request
final List<String> contentFilters = super.getLinkHandler().getContentFilters(); final List<String> contentFilters = super.getLinkHandler().getContentFilters();
@ -70,13 +73,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final byte[] body; final byte[] body;
if (!isNullOrEmpty(params)) { if (!isNullOrEmpty(params)) {
body = JsonWriter.string(prepareJsonBuilder() body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("query", query) .value("query", query)
.value("params", params) .value("params", params)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
} else { } else {
body = JsonWriter.string(prepareJsonBuilder() body = JsonWriter.string(prepareJsonBuilder(contentCountry)
.value("query", query) .value("query", query)
.done()) .done())
.getBytes(UTF_8); .getBytes(UTF_8);
@ -100,11 +103,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getObject("itemSectionRenderer"); .getObject("itemSectionRenderer");
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0)
.getObject("didYouMeanRenderer"); .getObject("didYouMeanRenderer");
final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents")
.getObject(0)
.getObject("showingResultsForRenderer"); .getObject("showingResultsForRenderer");
if (!didYouMeanRenderer.isEmpty()) { if (!didYouMeanRenderer.isEmpty()) {
return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); return JsonUtils.getString(didYouMeanRenderer,
"correctedQueryEndpoint.searchEndpoint.query");
} else if (showingResultsForRenderer != null) { } else if (showingResultsForRenderer != null) {
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
} else { } else {
@ -126,7 +131,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException { public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo( return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents")); .getObject("primaryContents").getObject("sectionListRenderer")
.getArray("contents"));
} }
@Nonnull @Nonnull
@ -134,20 +140,23 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer") final JsonArray sections = initialData.getObject("contents")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"); .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents");
Page nextPage = null; Page nextPage = null;
for (final Object section : sections) { for (final Object section : sections) {
if (((JsonObject) section).has("itemSectionRenderer")) { if (((JsonObject) section).has("itemSectionRenderer")) {
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer"); final JsonObject itemSectionRenderer = ((JsonObject) section)
.getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations")); nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
} else if (((JsonObject) section).has("continuationItemRenderer")) { } else if (((JsonObject) section).has("continuationItemRenderer")) {
nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer")); nextPage = getNewNextPageFrom(((JsonObject) section)
.getObject("continuationItemRenderer"));
} }
} }
@ -155,7 +164,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
} }
@Override @Override
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException { public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) { if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL"); throw new IllegalArgumentException("Page doesn't contain an URL");
} }
@ -174,29 +184,15 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
} else { } else {
// @formatter:off // @formatter:off
final byte[] json = JsonWriter.string() final byte[] json = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.object() .getCountryCode())
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.value("utcOffsetMinutes", 0)
.end()
.object("request").end()
.object("user").end()
.end()
.value("continuation", page.getId()) .value("continuation", page.getId())
.end().done().getBytes(UTF_8); .done())
.getBytes(UTF_8);
// @formatter:on // @formatter:on
final Map<String, List<String>> headers = new HashMap<>(); final String responseBody = getValidJsonResponseBody(getDownloader().post(
headers.put("Origin", Collections.singletonList("https://www.youtube.com")); page.getUrl(), new HashMap<>(), json));
headers.put("Referer", Collections.singletonList(this.getUrl()));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final JsonObject ajaxJson; final JsonObject ajaxJson;
try { try {
@ -206,16 +202,21 @@ public class YoutubeSearchExtractor extends SearchExtractor {
} }
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
.getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems"); .getObject(0).getObject("appendContinuationItemsAction")
.getArray("continuationItems");
final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents"); final JsonArray contents = continuationItems.getObject(0)
.getObject("itemSectionRenderer").getArray("contents");
collectStreamsFrom(collector, contents); collectStreamsFrom(collector, contents);
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer"))); return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1)
.getObject("continuationItemRenderer")));
} }
} }
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray contents) throws NothingFoundException, ParsingException { private void collectStreamsFrom(final InfoItemsSearchCollector collector,
final JsonArray contents) throws NothingFoundException,
ParsingException {
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object content : contents) { for (Object content : contents) {
@ -224,11 +225,14 @@ public class YoutubeSearchExtractor extends SearchExtractor {
throw new NothingFoundException(getTextFromObject( throw new NothingFoundException(getTextFromObject(
item.getObject("backgroundPromoRenderer").getObject("bodyText"))); item.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (item.has("videoRenderer")) { } else if (item.has("videoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(item.getObject("videoRenderer"), timeAgoParser)); collector.commit(new YoutubeStreamInfoItemExtractor(item
.getObject("videoRenderer"), timeAgoParser));
} else if (item.has("channelRenderer")) { } else if (item.has("channelRenderer")) {
collector.commit(new YoutubeChannelInfoItemExtractor(item.getObject("channelRenderer"))); collector.commit(new YoutubeChannelInfoItemExtractor(item
.getObject("channelRenderer")));
} else if (item.has("playlistRenderer")) { } else if (item.has("playlistRenderer")) {
collector.commit(new YoutubePlaylistInfoItemExtractor(item.getObject("playlistRenderer"))); collector.commit(new YoutubePlaylistInfoItemExtractor(item
.getObject("playlistRenderer")));
} }
} }
} }
@ -238,15 +242,18 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return null; return null;
} }
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); final JsonObject nextContinuationData = continuations.getObject(0)
.getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation"); final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); final String clickTrackingParams = nextContinuationData
.getString("clickTrackingParams");
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation="
+ "&itct=" + clickTrackingParams); + continuation + "&itct=" + clickTrackingParams);
} }
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuationItemRenderer)) { if (isNullOrEmpty(continuationItemRenderer)) {
return null; return null;
} }

322
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,6 +4,11 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
@ -12,8 +17,15 @@ import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
import org.schabi.newpipe.extractor.exceptions.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.Localization;
@ -120,8 +132,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable @Nullable
@Override @Override
public String getTextualUploadDate() throws ParsingException { public String getTextualUploadDate() throws ParsingException {
final JsonObject micro = final JsonObject micro = playerResponse.getObject("microformat")
playerResponse.getObject("microformat").getObject("playerMicroformatRenderer"); .getObject("playerMicroformatRenderer");
if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) { if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) {
return micro.getString("uploadDate"); return micro.getString("uploadDate");
} else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) { } else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) {
@ -140,11 +152,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) { if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10); .startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
.substring(10);
try { // Premiered 20 hours ago try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
Localization.fromLocalizationCode("en"));
OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime(); OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime); return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (final Exception ignored) { } catch (final Exception ignored) {
@ -159,8 +174,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
try { try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date // TODO: this parses English formatted dates only, we need a better approach to parse
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")), // the textual date
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer()
.getObject("dateText")),
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (final Exception ignored) { } catch (final Exception ignored) {
@ -185,7 +202,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
try { try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails"); JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail")
.getArray("thumbnails");
// the last thumbnail is the one with the highest resolution // the last thumbnail is the one with the highest resolution
String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
@ -202,13 +220,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched(); assertPageFetched();
// description with more info on links // description with more info on links
try { try {
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true); String description = getTextFromObject(getVideoSecondaryInfoRenderer()
.getObject("description"), true);
if (!isNullOrEmpty(description)) return new Description(description, Description.HTML); if (!isNullOrEmpty(description)) return new Description(description, Description.HTML);
} catch (final ParsingException ignored) { } catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here // age-restricted videos cause a ParsingException here
} }
String description = playerResponse.getObject("videoDetails").getString("shortDescription"); String description = playerResponse.getObject("videoDetails")
.getString("shortDescription");
if (description == null) { if (description == null) {
final JsonObject descriptionObject = playerResponse.getObject("microformat") final JsonObject descriptionObject = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer").getObject("description"); .getObject("playerMicroformatRenderer").getObject("description");
@ -289,22 +309,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public long getViewCount() throws ParsingException { public long getViewCount() throws ParsingException {
assertPageFetched(); assertPageFetched();
String views = null; final String views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count");
try {
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
if (isNullOrEmpty(views)) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (isNullOrEmpty(views)) throw new ParsingException("Could not get view count");
}
if (views.toLowerCase().contains("no views")) return 0;
return Long.parseLong(Utils.removeNonDigitCharacters(views)); return Long.parseLong(Utils.removeNonDigitCharacters(views));
} }
@ -320,13 +326,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (final NullPointerException e) { } catch (final NullPointerException e) {
// if this kicks in our button has no content and therefore ratings must be disabled // if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the like button is missing", e); throw new ParsingException(
"Ratings are enabled even though the like button is missing", e);
} }
return -1; return -1;
} }
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString)); return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (final NumberFormatException nfe) { } catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe); throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer",
nfe);
} catch (final Exception e) { } catch (final Exception e) {
if (getAgeLimit() == NO_AGE_LIMIT) { if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get like count", e); throw new ParsingException("Could not get like count", e);
@ -338,6 +346,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public long getDislikeCount() throws ParsingException { public long getDislikeCount() throws ParsingException {
assertPageFetched(); assertPageFetched();
String dislikesString = ""; String dislikesString = "";
try { try {
try { try {
@ -346,13 +355,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (final NullPointerException e) { } catch (final NullPointerException e) {
// if this kicks in our button has no content and therefore ratings must be disabled // if this kicks in our button has no content and therefore ratings must be disabled
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) { if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e); throw new ParsingException(
"Ratings are enabled even though the dislike button is missing", e);
} }
return -1; return -1;
} }
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString)); return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (final NumberFormatException nfe) { } catch (final NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe); throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer",
nfe);
} catch (final Exception e) { } catch (final Exception e) {
if (getAgeLimit() == NO_AGE_LIMIT) { if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get dislike count", e); throw new ParsingException("Could not get dislike count", e);
@ -366,16 +377,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
assertPageFetched(); assertPageFetched();
try {
final String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (!isNullOrEmpty(uploaderUrl)) {
return uploaderUrl;
}
} catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here
}
final String uploaderId = playerResponse.getObject("videoDetails").getString("channelId"); final String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (!isNullOrEmpty(uploaderId)) { if (!isNullOrEmpty(uploaderId)) {
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId); return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
@ -389,19 +390,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
assertPageFetched(); assertPageFetched();
String uploaderName = null; // Don't use the name in the videoSecondaryRenderer object to get real name of the uploader
// The difference between the real name of the channel and the displayed name is especially
try { // visible for music channels and autogenerated channels.
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") final String uploaderName = playerResponse.getObject("videoDetails").getString("author");
.getObject("videoOwnerRenderer").getObject("title")); if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name");
} catch (final ParsingException ignored) {
}
if (isNullOrEmpty(uploaderName)) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (isNullOrEmpty(uploaderName)) throw new ParsingException("Could not get uploader name");
}
return uploaderName; return uploaderName;
} }
@ -422,8 +415,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String url = null; String url = null;
try { try {
url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer") url = getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); .getObject("videoOwnerRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (final ParsingException ignored) { } catch (final ParsingException ignored) {
// age-restricted videos cause a ParsingException here // age-restricted videos cause a ParsingException here
} }
@ -471,11 +465,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
if (!dashManifestUrl.contains("/signature/")) { if (!dashManifestUrl.contains("/signature/")) {
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)",
dashManifestUrl);
final String deobfuscatedSig; final String deobfuscatedSig;
deobfuscatedSig = deobfuscateSignature(obfuscatedSig); deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig); dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig,
"/signature/" + deobfuscatedSig);
} }
return dashManifestUrl; return dashManifestUrl;
@ -503,7 +499,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try { try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.AUDIO).entrySet()) {
final ItagItem itag = entry.getValue(); final ItagItem itag = entry.getValue();
String url = entry.getKey(); String url = entry.getKey();
url = throttlingDecrypter.apply(url); url = throttlingDecrypter.apply(url);
@ -527,7 +524,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try { try {
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
ItagItem.ItagType.VIDEO).entrySet()) {
final ItagItem itag = entry.getValue(); final ItagItem itag = entry.getValue();
String url = entry.getKey(); String url = entry.getKey();
url = throttlingDecrypter.apply(url); url = throttlingDecrypter.apply(url);
@ -551,7 +549,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId()); final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try { try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
final ItagItem itag = entry.getValue(); final ItagItem itag = entry.getValue();
String url = entry.getKey(); String url = entry.getKey();
url = throttlingDecrypter.apply(url); url = throttlingDecrypter.apply(url);
@ -620,27 +619,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM; ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM;
} }
@Nullable
private StreamInfoItemExtractor getNextStream() throws ExtractionException {
try {
final JsonObject firstWatchNextItem = initialData.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
.getObject("secondaryResults").getArray("results").getObject(0);
if (!firstWatchNextItem.has("compactAutoplayRenderer")) {
// there is no "next" stream
return null;
}
final JsonObject videoInfo = firstWatchNextItem.getObject("compactAutoplayRenderer")
.getArray("contents").getObject(0).getObject("compactVideoRenderer");
return new YoutubeStreamInfoItemExtractor(videoInfo, getTimeAgoParser());
} catch (final Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
@Nullable @Nullable
@Override @Override
public StreamInfoItemsCollector getRelatedItems() throws ExtractionException { public StreamInfoItemsCollector getRelatedItems() throws ExtractionException {
@ -651,21 +629,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
try { try {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
getServiceId());
final StreamInfoItemExtractor nextStream = getNextStream(); final JsonArray results = initialData.getObject("contents")
if (nextStream != null) { .getObject("twoColumnWatchNextResults").getObject("secondaryResults")
collector.commit(nextStream); .getObject("secondaryResults").getArray("results");
}
final JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("secondaryResults").getObject("secondaryResults").getArray("results");
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object ul : results) { for (final Object ul : results) {
if (((JsonObject) ul).has("compactVideoRenderer")) { if (((JsonObject) ul).has("compactVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul).getObject("compactVideoRenderer"), timeAgoParser)); collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) ul)
.getObject("compactVideoRenderer"), timeAgoParser));
} }
} }
return collector; return collector;
@ -680,9 +656,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getErrorMessage() { public String getErrorMessage() {
try { try {
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse") return getTextFromObject(playerResponse.getObject("playabilityStatus")
.getObject("playabilityStatus").getObject("errorScreen") .getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("playerErrorMessageRenderer").getObject("reason")); .getObject("reason"));
} catch (final ParsingException | NullPointerException e) { } catch (final ParsingException | NullPointerException e) {
return null; // no error message return null; // no error message
} }
@ -737,7 +713,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
youtubePlayerResponse = playerResponse; youtubePlayerResponse = playerResponse;
} }
JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse : playerResponse) JsonObject playabilityStatus = (playerResponse == null ? youtubePlayerResponse
: playerResponse)
.getObject("playabilityStatus"); .getObject("playabilityStatus");
String status = playabilityStatus.getString("status"); String status = playabilityStatus.getString("status");
// If status exist, and is not "OK", throw the specific exception based on error message // If status exist, and is not "OK", throw the specific exception based on error message
@ -755,8 +732,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PrivateContentException("This video is private."); throw new PrivateContentException("This video is private.");
} }
} else if (reason.equals("Sign in to confirm your age")) { } else if (reason.equals("Sign in to confirm your age")) {
// No streams can be fetched, therefore thrown an AgeRestrictedContentException explicitly. // No streams can be fetched, therefore thrown an AgeRestrictedContentException
throw new AgeRestrictedContentException("This age-restricted video cannot be watched."); // explicitly.
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched.");
} }
} }
if (status.equalsIgnoreCase("unplayable")) { if (status.equalsIgnoreCase("unplayable")) {
@ -772,7 +751,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new PaidContentException("This video is only available for members of the channel of this video"); throw new PaidContentException("This video is only available for members of the channel of this video");
} }
if (reason.equals("Video unavailable")) { if (reason.equals("Video unavailable")) {
final String detailedErrorMessage = playabilityStatus.getObject("errorScreen") final String detailedErrorMessage = playabilityStatus
.getObject("errorScreen")
.getObject("playerErrorMessageRenderer") .getObject("playerErrorMessageRenderer")
.getObject("subreason") .getObject("subreason")
.getArray("runs") .getArray("runs")
@ -780,7 +760,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
.getString("text"); .getString("text");
if (detailedErrorMessage != null) { if (detailedErrorMessage != null) {
if (detailedErrorMessage.equals("The uploader has not made this video available in your country.")) { if (detailedErrorMessage.equals("The uploader has not made this video available in your country.")) {
throw new GeographicRestrictionException("This video is not available in user's country."); throw new GeographicRestrictionException(
"This video is not available in user's country.");
} }
} }
} }
@ -792,9 +773,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException { private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
final String videoInfoUrl = getVideoInfoUrl(getId()); final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization()); final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody())); final String infoPageResponse = NewPipe.getDownloader()
.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
try { try {
playerResponse = JsonParser.object().from(videoInfoPage.get("player_response")); playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
@ -804,6 +787,39 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
@Nonnull
private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
try {
// Don't provide a video id to get a smaller response (around 9kb instead of 21 kb)
final String embedUrl = "https://www.youtube.com/embed/";
final String embedPageContent = NewPipe.getDownloader()
.get(embedUrl, getExtractorLocalization()).responseBody();
try {
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
} catch (final Parser.RegexException ex) {
// playerJsUrl is still available in the file, just somewhere else TODO
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (final Element elem : elems) {
if (elem.attr("src").contains("base.js")) {
playerJsUrl = elem.attr("src");
break;
}
}
}
// Get embed sts
return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent);
} catch (final Exception i) {
// if it fails we simply reply with no sts as then it does not seem to be necessary
return "";
}
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException { private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null; Parser.RegexException exception = null;
for (final String regex : REGEXES) { for (final String regex : REGEXES) {
@ -815,7 +831,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
} }
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception); throw new DeobfuscateException(
"Could not find deobfuscate function with any of the given patterns.", exception);
} }
private String loadDeobfuscationCode() private String loadDeobfuscationCode()
@ -827,17 +844,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String functionPattern = "(" final String functionPattern = "("
+ deobfuscationFunctionName.replace("$", "\\$") + deobfuscationFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern,
playerCode) + ";";
final String helperObjectName = final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction); Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(",
deobfuscateFunction);
final String helperPattern = final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; "(var " + helperObjectName.replace("$", "\\$")
+ "=\\{.+?\\}\\};)";
final String helperObject = final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
final String callerFunction = final String callerFunction =
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}"; "function " + DEOBFUSCATION_FUNC_NAME + "(a){return "
+ deobfuscationFunctionName + "(a);}";
return helperObject + deobfuscateFunction + callerFunction; return helperObject + deobfuscateFunction + callerFunction;
} catch (final Exception e) { } catch (final Exception e) {
@ -879,13 +900,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException { private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer; if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") final JsonArray contents = initialData.getObject("contents")
.getObject("results").getObject("results").getArray("contents"); .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoPrimaryInfoRenderer = null; JsonObject videoPrimaryInfoRenderer = null;
for (final Object content : contents) { for (final Object content : contents) {
if (((JsonObject) content).has("videoPrimaryInfoRenderer")) { if (((JsonObject) content).has("videoPrimaryInfoRenderer")) {
videoPrimaryInfoRenderer = ((JsonObject) content).getObject("videoPrimaryInfoRenderer"); videoPrimaryInfoRenderer = ((JsonObject) content)
.getObject("videoPrimaryInfoRenderer");
break; break;
} }
} }
@ -901,13 +924,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer; if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults") final JsonArray contents = initialData.getObject("contents")
.getObject("results").getObject("results").getArray("contents"); .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
JsonObject videoSecondaryInfoRenderer = null; JsonObject videoSecondaryInfoRenderer = null;
for (final Object content : contents) { for (final Object content : contents) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
videoSecondaryInfoRenderer = ((JsonObject) content).getObject("videoSecondaryInfoRenderer"); videoSecondaryInfoRenderer = ((JsonObject) content)
.getObject("videoSecondaryInfoRenderer");
break; break;
} }
} }
@ -921,11 +946,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Nonnull @Nonnull
private static String getVideoInfoUrl(final String id) { private static String getVideoInfoUrl(final String id, final String sts) {
// TODO: Try parsing embedded_player_response first // TODO: Try parsing embedded_player_response first
return "https://www.youtube.com/get_video_info?video_id=" + id + return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&eurl=https://youtube.googleapis.com/v/" + id + "&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
"&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en"; "&sts=" + sts + "&ps=default&gl=US&hl=en";
} }
private Map<String, ItagItem> getItags(final String streamingDataKey, private Map<String, ItagItem> getItags(final String streamingDataKey,
@ -969,15 +994,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
JsonObject initRange = formatData.getObject("initRange"); JsonObject initRange = formatData.getObject("initRange");
JsonObject indexRange = formatData.getObject("indexRange"); JsonObject indexRange = formatData.getObject("indexRange");
String mimeType = formatData.getString("mimeType", EMPTY_STRING); String mimeType = formatData.getString("mimeType", EMPTY_STRING);
String codec = mimeType.contains("codecs") ? mimeType.split("\"")[1] : EMPTY_STRING; String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate")); itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width")); itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height")); itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1"))); itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1"))); "-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1"))); itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1"))); "-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
"-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
"-1")));
itagItem.fps = formatData.getInt("fps"); itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality")); itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec); itagItem.setCodec(codec);
@ -1027,10 +1057,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final int totalCount = Integer.parseInt(parts[2]); final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]); final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]); final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7]; final String baseUrl = url.replace("$L", String.valueOf(i - 1))
.replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls; final List<String> urls;
if (baseUrl.contains("$M")) { if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY)); final int totalPages = (int) Math.ceil(totalCount / (double)
(framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages); urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) { for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j))); urls.add(baseUrl.replace("$M", String.valueOf(j)));
@ -1064,26 +1096,24 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public Privacy getPrivacy() { public Privacy getPrivacy() {
final boolean isUnlisted = playerResponse final boolean isUnlisted = playerResponse.getObject("microformat")
.getObject("microformat") .getObject("playerMicroformatRenderer").getBoolean("isUnlisted");
.getObject("playerMicroformatRenderer")
.getBoolean("isUnlisted");
return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC; return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
} }
@Nonnull @Nonnull
@Override @Override
public String getCategory() { public String getCategory() {
return playerResponse.getObject("microformat") return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer")
.getObject("playerMicroformatRenderer") .getString("category");
.getString("category", EMPTY_STRING);
} }
@Nonnull @Nonnull
@Override @Override
public String getLicence() throws ParsingException { public String getLicence() throws ParsingException {
final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer").getArray("rows") .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getArray("rows")
.getObject(0).getObject("metadataRowRenderer"); .getObject(0).getObject("metadataRowRenderer");
final JsonArray contents = metadataRowRenderer.getArray("contents"); final JsonArray contents = metadataRowRenderer.getArray("contents");
@ -1099,7 +1129,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull @Nonnull
@Override @Override
public List<String> getTags() { public List<String> getTags() {
return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails").getArray("keywords")); return JsonUtils.getStringListFromJsonArray(playerResponse.getObject("videoDetails")
.getArray("keywords"));
} }
@Nonnull @Nonnull
@ -1118,12 +1149,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Search for correct panel containing the data // Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) { for (int i = 0; i < panels.size(); i++) {
final String panelIdentifier = panels.getObject(i).getObject("engagementPanelSectionListRenderer") final String panelIdentifier = panels.getObject(i)
.getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier"); .getString("panelIdentifier");
if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters") if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters")
|| panelIdentifier.equals("engagement-panel-macro-markers")) { || panelIdentifier.equals("engagement-panel-macro-markers")) {
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer") segmentsArray = panels.getObject(i)
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents"); .getObject("engagementPanelSectionListRenderer").getObject("content")
.getObject("macroMarkersListRenderer").getArray("contents");
break; break;
} }
} }
@ -1131,9 +1164,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (segmentsArray != null) { if (segmentsArray != null) {
final long duration = getLength(); final long duration = getLength();
for (final Object object : segmentsArray) { for (final Object object : segmentsArray) {
final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer"); final JsonObject segmentJson = ((JsonObject) object)
.getObject("macroMarkersListItemRenderer");
final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint") final int startTimeSeconds = segmentJson.getObject("onTap")
.getObject("watchEndpoint")
.getInt("startTimeSeconds", -1); .getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) { if (startTimeSeconds == -1) {
@ -1151,10 +1186,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final StreamSegment segment = new StreamSegment(title, startTimeSeconds); final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds); segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) { if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails"); final JsonArray previewsArray = segmentJson.getObject("thumbnail")
.getArray("thumbnails");
if (!previewsArray.isEmpty()) { if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the last position // Assume that the thumbnail with the highest resolution is at the
final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url"); // last position
final String url = previewsArray
.getObject(previewsArray.size() - 1).getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url)); segment.setPreviewUrl(fixThumbnailUrl(url));
} }
} }

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -42,6 +42,7 @@ import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -57,18 +58,11 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override @Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
// @formatter:off // @formatter:off
final byte[] body = JsonWriter.string() final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorContentCountry()
.object() .getCountryCode())
.object("context")
.object("client")
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("browseId", "FEtrending") .value("browseId", "FEtrending")
.end().done().getBytes(UTF_8); .done())
.getBytes(UTF_8);
// @formatter:on // @formatter:on
initialData = getJsonPostResponse("browse", body, getExtractorLocalization()); initialData = getJsonPostResponse("browse", body, getExtractorLocalization());