CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #262 from wb9688/pbj

Browse source 
Improve yt_new
This commit is contained in:
Tobias Groza 2020-02-29 21:39:02 +01:00 committed by GitHub
commit 5a101fd17f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 605 additions and 554 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file

@ -2,33 +2,27 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -52,11 +46,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
@SuppressWarnings("WeakerAccess")
public class YoutubeChannelExtractor extends ChannelExtractor {
/*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/";
private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
private Document doc;
private JsonObject initialData;
private JsonObject videoTab;
public YoutubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
@ -64,23 +55,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
String channelUrl = super.getUrl() + CHANNEL_URL_PARAMETERS;
final Response response = downloader.get(channelUrl, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(channelUrl, response);
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("continuations"));
if (getVideoTab() == null) return "";
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations"));
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
try {
return CHANNEL_URL_BASE + getId();
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + getId());
} catch (ParsingException e) {
return super.getUrl();
}
@ -109,8 +104,10 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getAvatarUrl() throws ParsingException {
try {
return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("avatar")
.getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get avatar", e);
}
@ -127,17 +124,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null;
}
// the first characters of the banner URLs are different for each channel and some are not even valid URLs
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get banner", e);
}
@ -157,13 +145,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) {
try {
return Utils.mixedNumberWordToLong(subscriberInfo.getArray("runs").getObject(0).getString("text"));
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
} catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
// If the element is null, the channel have the subscriber count disabled
return -1;
// If there's no subscribe button, the channel has the subscriber count disabled
if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
return -1;
} else {
return 0;
}
}
}
@ -181,8 +173,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents");
collectStreamsFrom(collector, videos);
if (getVideoTab() != null) {
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents")
.getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("gridRenderer").getArray("items");
collectStreamsFrom(collector, videos);
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@ -198,46 +194,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray ajaxJson;
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("sectionListContinuation");
.getObject("continuationContents").getObject("gridContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
}
private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) {
return "";
}
if (continuations == null) return "";
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
@ -254,10 +223,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
JsonObject videoInfo = ((JsonObject) video).getObject("itemSectionRenderer")
.getArray("contents").getObject(0);
if (videoInfo.getObject("videoRenderer") != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo.getObject("videoRenderer"), timeAgoParser) {
if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
@ -273,6 +241,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab;
JsonArray tabs = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs");
JsonObject videoTab = null;
@ -290,6 +260,15 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ParsingException("Could not find Videos tab");
}
try {
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
} catch (Exception ignored) {}
this.videoTab = videoTab;
return videoTab;
}
}

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View file

@ -7,8 +7,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/*
* Created by Christian Schabesberger on 12.02.17.
@ -41,15 +41,8 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
public String getThumbnailUrl() throws ParsingException {
try {
String url = channelInfoItem.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
if (url.startsWith("//")) {
url = url.substring(2);
}
if (url.startsWith(HTTP)) {
url = Utils.replaceHttpWithHttps(url);
} else if (!url.startsWith(HTTPS)) {
url = HTTPS + url;
}
return url;
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
@ -58,7 +51,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getName() throws ParsingException {
try {
return channelInfoItem.getObject("title").getString("simpleText");
return getTextFromObject(channelInfoItem.getObject("title"));
} catch (Exception e) {
throw new ParsingException("Could not get name", e);
}
@ -67,7 +60,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getUrl() throws ParsingException {
try {
String id = "channel/" + channelInfoItem.getString("channelId"); // Does prepending 'channel/' always work?
String id = "channel/" + channelInfoItem.getString("channelId");
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) {
throw new ParsingException("Could not get url", e);
@ -77,7 +70,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getSubscriberCount() throws ParsingException {
try {
String subscribers = channelInfoItem.getObject("subscriberCountText").getString("simpleText").split(" ")[0];
String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
return Utils.mixedNumberWordToLong(subscribers);
} catch (Exception e) {
throw new ParsingException("Could not get subscriber count", e);
@ -87,8 +80,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public long getStreamCount() throws ParsingException {
try {
return Long.parseLong(Utils.removeNonDigitCharacters(channelInfoItem.getObject("videoCountText")
.getArray("runs").getObject(0).getString("text")));
return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
} catch (Exception e) {
throw new ParsingException("Could not get stream count", e);
}
@ -97,7 +89,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
@Override
public String getDescription() throws ParsingException {
try {
return channelInfoItem.getObject("descriptionSnippet").getArray("runs").getObject(0).getString("text");
return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
} catch (Exception e) {
throw new ParsingException("Could not get description", e);
}

86
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -2,37 +2,30 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc;
private JsonObject initialData;
private JsonObject uploaderInfo;
private JsonObject playlistInfo;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
@ -41,11 +34,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
uploaderInfo = getUploaderInfo();
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
playlistInfo = getPlaylistInfo();
}
@ -94,7 +87,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
try {
String name = playlistInfo.getObject("title").getArray("runs").getObject(0).getString("text");
String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null) return name;
} catch (Exception ignored) {}
try {
@ -106,16 +99,23 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getThumbnailUrl() throws ParsingException {
String url = null;
try {
return playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
try {
return initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception e) {
throw new ParsingException("Could not get playlist thumbnail", e);
if (url == null) {
try {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get playlist thumbnail");
}
return fixThumbnailUrl(url);
}
@Override
@ -127,8 +127,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
try {
return YoutubeChannelExtractor.CHANNEL_URL_BASE +
uploaderInfo.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId");
return getUrlFromNavigationEndpoint(getUploaderInfo().getObject("navigationEndpoint"));
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader url", e);
}
@ -137,7 +136,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderName() throws ParsingException {
try {
return uploaderInfo.getObject("title").getArray("runs").getObject(0).getString("text");
return getTextFromObject(getUploaderInfo().getObject("title"));
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader name", e);
}
@ -146,7 +145,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
return uploaderInfo.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get playlist uploader avatar", e);
}
@ -155,7 +156,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public long getStreamCount() throws ParsingException {
try {
String viewsText = getPlaylistInfo().getArray("stats").getObject(0).getArray("runs").getObject(0).getString("text");
String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) {
throw new ParsingException("Could not get video count from playlist", e);
@ -184,32 +185,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray ajaxJson;
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java
View file

@ -7,6 +7,9 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
private JsonObject playlistInfoItem;
@ -17,8 +20,10 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
return playlistInfoItem.getArray("thumbnails").getObject(0).getArray("thumbnails")
.getObject(0).getString("url");
String url = playlistInfoItem.getArray("thumbnails").getObject(0)
.getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
@ -27,7 +32,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getName() throws ParsingException {
try {
return playlistInfoItem.getObject("title").getString("simpleText");
return getTextFromObject(playlistInfoItem.getObject("title"));
} catch (Exception e) {
throw new ParsingException("Could not get name", e);
}
@ -46,7 +51,7 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract
@Override
public String getUploaderName() throws ParsingException {
try {
return playlistInfoItem.getObject("longBylineText").getArray("runs").getObject(0).getString("text");
return getTextFromObject(playlistInfoItem.getObject("longBylineText"));
} catch (Exception e) {
throw new ParsingException("Could not get uploader name", e);
}

68
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View file

@ -2,30 +2,24 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
/*
* Created by Christian Schabesberger on 22.07.2018
*
@ -47,8 +41,6 @@ import javax.annotation.Nonnull;
*/
public class YoutubeSearchExtractor extends SearchExtractor {
private Document doc;
private JsonObject initialData;
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
@ -57,10 +49,11 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl();
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response);
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
}
@Nonnull
@ -79,8 +72,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
if (showingResultsForRenderer == null) {
return "";
} else {
return showingResultsForRenderer.getObject("correctedQuery").getArray("runs")
.getObject(0).getString("text");
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
}
}
@ -88,11 +80,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray videos = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents")
.getObject(0).getObject("itemSectionRenderer").getArray("contents");
JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
for (Object section : sections) {
collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents"));
}
collectStreamsFrom(collector, videos);
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@ -110,33 +104,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
InfoItemsSearchCollector collector = getInfoItemSearchCollector();
JsonArray ajaxJson;
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
}
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
@ -153,8 +121,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
for (Object item : videos) {
if (((JsonObject) item).getObject("backgroundPromoRenderer") != null) {
throw new NothingFoundException(((JsonObject) item).getObject("backgroundPromoRenderer")
.getObject("bodyText").getArray("runs").getObject(0).getString("text"));
throw new NothingFoundException(getTextFromObject(((JsonObject) item)
.getObject("backgroundPromoRenderer").getObject("bodyText")));
} else if (((JsonObject) item).getObject("videoRenderer") != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) item).getObject("videoRenderer"), timeAgoParser));
} else if (((JsonObject) item).getObject("channelRenderer") != null) {

274
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -4,8 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
@ -13,8 +11,6 @@ import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -24,6 +20,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
@ -40,8 +37,6 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
@ -56,6 +51,11 @@ import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/*
* Created by Christian Schabesberger on 06.08.15.
*
@ -89,19 +89,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
private Document doc;
private JsonArray initialAjaxJson;
@Nullable
private JsonObject playerArgs;
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject initialData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit;
@Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
private boolean isAgeRestricted;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@ -115,16 +116,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getName() throws ParsingException {
assertPageFetched();
String title = null;
try {
title = getVideoPrimaryInfoRenderer().getObject("title").getArray("runs").getObject(0).getString("text");
title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
} catch (Exception ignored) {}
if (title == null) {
try {
title = playerResponse.getObject("videoDetails").getString("title");
} catch (Exception ignored) {}
if (title == null) throw new ParsingException("Could not get name");
}
if (title != null) return title;
throw new ParsingException("Could not get name");
return title;
}
@Override
@ -144,8 +149,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (Exception ignored) {}
try {
if (getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").startsWith("Premiered")) {
String time = getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText").substring(10);
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
String time = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).substring(10);
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
@ -163,7 +168,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
// TODO this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getVideoPrimaryInfoRenderer().getObject("dateText").getString("simpleText"));
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {}
throw new ParsingException("Could not get upload date");
@ -187,8 +192,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
return thumbnails.getObject(thumbnails.size() - 1).getString("url");
String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url");
}
@ -201,55 +207,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
// description with more info on links
try {
boolean htmlConversionRequired = false;
JsonArray descriptions = getVideoSecondaryInfoRenderer().getObject("description").getArray("runs");
StringBuilder descriptionBuilder = new StringBuilder(descriptions.size());
for (Object textObjectHolder : descriptions) {
JsonObject textHolder = (JsonObject) textObjectHolder;
String text = textHolder.getString("text");
if (textHolder.getObject("navigationEndpoint") != null) {
// The text is a link. Get the URL it points to and generate a HTML link of it
if (textHolder.getObject("navigationEndpoint").getObject("urlEndpoint") != null) {
String internUrl = textHolder.getObject("navigationEndpoint").getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
if (url != null && !url.isEmpty()) {
descriptionBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
} else {
descriptionBuilder.append(text);
}
break;
}
}
} else if (internUrl.startsWith("http")) {
descriptionBuilder.append("<a href=\"").append(internUrl).append("\">").append(text).append("</a>");
htmlConversionRequired = true;
}
continue;
}
continue;
}
if (text != null) {
descriptionBuilder.append(text);
}
}
String description = descriptionBuilder.toString();
if (!description.isEmpty()) {
if (htmlConversionRequired) {
description = description.replaceAll("\\n", "<br>");
description = description.replaceAll(" ", " &nbsp;");
return new Description(description, Description.HTML);
}
return new Description(description, Description.PLAIN_TEXT);
}
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
return new Description(description, Description.HTML);
} catch (Exception ignored) { }
// raw non-html description
@ -261,17 +220,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public int getAgeLimit() throws ParsingException {
assertPageFetched();
if (!isAgeRestricted) {
return NO_AGE_LIMIT;
}
try {
return Integer.valueOf(doc.select("meta[property=\"og:restrictions:age\"]")
.attr(CONTENT).replace("+", ""));
} catch (Exception e) {
throw new ParsingException("Could not get age restriction");
}
public int getAgeLimit() {
if (initialData == null || initialData.isEmpty()) throw new IllegalStateException("initialData is not parsed yet");
return ageLimit;
}
@Override
@ -311,24 +263,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public long getViewCount() throws ParsingException {
assertPageFetched();
String views = null;
try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount")
.getArray("runs").getObject(0).getString("text");
views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
} catch (Exception ignored) {}
if (views == null) {
try {
views = getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount").getString("simpleText");
} catch (Exception ignored) {}
}
if (views == null) {
try {
views = playerResponse.getObject("videoDetails").getString("viewCount");
} catch (Exception ignored) {}
if (views == null) throw new ParsingException("Could not get view count");
}
if (views != null) return Long.parseLong(Utils.removeNonDigitCharacters(views));
throw new ParsingException("Could not get view count");
return Long.parseLong(Utils.removeNonDigitCharacters(views));
}
@Override
@ -381,17 +330,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getUploaderUrl() throws ParsingException {
assertPageFetched();
String uploaderId = null;
try {
uploaderId = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("navigationEndpoint").getObject("browseEndpoint").getString("browseId");
String uploaderUrl = getUrlFromNavigationEndpoint(getVideoSecondaryInfoRenderer()
.getObject("owner").getObject("videoOwnerRenderer").getObject("navigationEndpoint"));
if (uploaderUrl != null) return uploaderUrl;
} catch (Exception ignored) {}
try {
String uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
if (uploaderId != null)
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("channel/" + uploaderId);
} catch (Exception ignored) {}
if (uploaderId == null) {
try {
uploaderId = playerResponse.getObject("videoDetails").getString("channelId");
} catch (Exception ignored) {}
}
if (uploaderId != null) return "https://www.youtube.com/channel/" + uploaderId;
throw new ParsingException("Could not get uploader url");
}
@ -400,44 +348,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getUploaderName() throws ParsingException {
assertPageFetched();
String uploaderName = null;
try {
uploaderName = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("title").getArray("runs").getObject(0).getString("text");
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (Exception ignored) {}
if (uploaderName == null) {
try {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
} catch (Exception ignored) {}
if (uploaderName == null) throw new ParsingException("Could not get uploader name");
}
if (uploaderName != null) return uploaderName;
throw new ParsingException("Could not get uploader name");
return uploaderName;
}
@Nonnull
@Override
public String getUploaderAvatarUrl() throws ParsingException {
assertPageFetched();
String uploaderAvatarUrl = null;
try {
uploaderAvatarUrl = initialData.getObject("contents").getObject("twoColumnWatchNextResults").getObject("secondaryResults")
.getObject("secondaryResults").getArray("results").getObject(0).getObject("compactAutoplayRenderer")
.getArray("contents").getObject(0).getObject("compactVideoRenderer").getObject("channelThumbnail")
.getArray("thumbnails").getObject(0).getString("url");
if (uploaderAvatarUrl != null && !uploaderAvatarUrl.isEmpty()) {
return uploaderAvatarUrl;
}
} catch (Exception ignored) {}
try {
uploaderAvatarUrl = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
String url = getVideoSecondaryInfoRenderer().getObject("owner").getObject("videoOwnerRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}
if (uploaderAvatarUrl == null) {
throw new ParsingException("Could not get uploader avatar url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get uploader avatar url", e);
}
return uploaderAvatarUrl;
}
@Nonnull
@ -578,9 +517,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public StreamInfoItem getNextStream() throws ExtractionException {
assertPageFetched();
if (isAgeRestricted) {
return null;
}
if (getAgeLimit() != NO_AGE_LIMIT) return null;
try {
final JsonObject videoInfo = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("secondaryResults").getObject("secondaryResults").getArray("results")
@ -599,9 +538,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
assertPageFetched();
if (isAgeRestricted) {
return null;
}
if (getAgeLimit() != NO_AGE_LIMIT) return null;
try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray results = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
@ -625,23 +564,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
*/
@Override
public String getErrorMessage() {
StringBuilder errorReason;
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
if (errorElement == null) {
errorReason = null;
} else {
String errorMessage = errorElement.text();
if (errorMessage == null || errorMessage.isEmpty()) {
errorReason = null;
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
}
return errorReason != null ? errorReason.toString() : "";
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
.getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
}
/*//////////////////////////////////////////////////////////////////////////
@ -651,11 +575,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
@ -667,32 +588,32 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private volatile String decryptionCode = "";
private String pageHtml = null;
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
pageHtml = response.responseBody();
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
final String url = getUrl() + "&pbj=1";
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
final String playerUrl;
// Check if the video is age restricted
if (!doc.select("meta[property=\"og:restrictions:age\"]").isEmpty()) {
if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
initialData = initialAjaxJson.getObject(2).getObject("response");
ageLimit = 18;
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
isAgeRestricted = true;
} else {
final JsonObject ytPlayerConfig = getPlayerConfig();
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
initialData = initialAjaxJson.getObject(3).getObject("response");
ageLimit = NO_AGE_LIMIT;
playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
}
playerResponse = getPlayerResponse();
initialData = YoutubeParsingHelper.getInitialData(pageHtml);
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
@ -703,21 +624,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private JsonObject getPlayerConfig() throws ParsingException {
try {
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
return JsonParser.object().from(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
if (errorReason.isEmpty()) {
throw new ContentNotAvailableException("Content not available: player config empty", e);
}
throw new ContentNotAvailableException("Content not available", e);
} catch (Exception e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
JsonObject playerArgs;
@ -869,7 +775,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
// If the video is age restricted getPlayerConfig will fail
if (isAgeRestricted) return Collections.emptyList();
if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
final JsonObject captions;
if (!playerResponse.has("captions")) {
@ -939,6 +845,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//////////////////////////////////////////////////////////////////////////*/
private JsonObject getVideoPrimaryInfoRenderer() throws ParsingException {
if (this.videoPrimaryInfoRenderer != null) return this.videoPrimaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents");
JsonObject videoPrimaryInfoRenderer = null;
@ -954,10 +862,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoPrimaryInfoRenderer");
}
this.videoPrimaryInfoRenderer = videoPrimaryInfoRenderer;
return videoPrimaryInfoRenderer;
}
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (this.videoSecondaryInfoRenderer != null) return this.videoSecondaryInfoRenderer;
JsonArray contents = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents");
JsonObject videoSecondaryInfoRenderer = null;
@ -973,6 +884,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
}
this.videoSecondaryInfoRenderer = videoSecondaryInfoRenderer;
return videoSecondaryInfoRenderer;
}
@ -1010,9 +922,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {
}
} catch (UnsupportedEncodingException ignored) {}
}
}
@ -1023,17 +933,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
int p = script.indexOf("ytplayer.config");
if (p == -1) {
return Collections.emptyList();
}
p = script.indexOf('{', p);
int e = script.indexOf("ytplayer.load", p);
if (e == -1) {
return Collections.emptyList();
}
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");

121
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View file

@ -6,7 +6,6 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
@ -15,6 +14,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeStreamInfoItemExtractor.java is part of NewPipe.
@ -76,89 +79,95 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getName() throws ParsingException {
String name = null;
try {
name = videoInfo.getObject("title").getString("simpleText");
} catch (Exception ignored) {}
if (name == null) {
try {
name = videoInfo.getObject("title").getArray("runs").getObject(0).getString("text");
} catch (Exception ignored) {}
}
String name = getTextFromObject(videoInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get name");
}
@Override
public long getDuration() throws ParsingException {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
String duration = null;
try {
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
return YoutubeParsingHelper.parseDurationString(videoInfo.getObject("lengthText").getString("simpleText"));
} catch (Exception e) {
throw new ParsingException("Could not get duration", e);
duration = getTextFromObject(videoInfo.getObject("lengthText"));
} catch (Exception ignored) {}
if (duration == null) {
try {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).getObject("thumbnailOverlayTimeStatusRenderer") != null) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
.getObject("thumbnailOverlayTimeStatusRenderer").getObject("text"));
}
}
} catch (Exception ignored) {}
if (duration == null) throw new ParsingException("Could not get duration");
}
return YoutubeParsingHelper.parseDurationString(duration);
}
@Override
public String getUploaderName() throws ParsingException {
String name = null;
try {
name = videoInfo.getObject("longBylineText").getArray("runs")
.getObject(0).getString("text");
name = getTextFromObject(videoInfo.getObject("longBylineText"));
} catch (Exception ignored) {}
if (name == null) {
try {
name = videoInfo.getObject("ownerText").getArray("runs")
.getObject(0).getString("text");
name = getTextFromObject(videoInfo.getObject("ownerText"));
} catch (Exception ignored) {}
if (name == null) {
try {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
} catch (Exception ignored) {}
if (name == null) throw new ParsingException("Could not get uploader name");
}
}
if (name == null) {
try {
name = videoInfo.getObject("shortBylineText").getArray("runs")
.getObject(0).getString("text");
} catch (Exception ignored) {}
}
if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get uploader name");
return name;
}
@Override
public String getUploaderUrl() throws ParsingException {
String url = null;
try {
String id = null;
url = getUrlFromNavigationEndpoint(videoInfo.getObject("longBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) {
try {
id = videoInfo.getObject("longBylineText").getArray("runs")
.getObject(0).getObject("navigationEndpoint")
.getObject("browseEndpoint").getString("browseId");
url = getUrlFromNavigationEndpoint(videoInfo.getObject("ownerText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (id == null) {
if (url == null) {
try {
id = videoInfo.getObject("ownerText").getArray("runs")
.getObject(0).getObject("navigationEndpoint")
.getObject("browseEndpoint").getString("browseId");
url = getUrlFromNavigationEndpoint(videoInfo.getObject("shortBylineText")
.getArray("runs").getObject(0).getObject("navigationEndpoint"));
} catch (Exception ignored) {}
if (url == null) throw new ParsingException("Could not get uploader url");
}
if (id == null) {
try {
id = videoInfo.getObject("shortBylineText").getArray("runs")
.getObject(0).getObject("navigationEndpoint")
.getObject("browseEndpoint").getString("browseId");
} catch (Exception ignored) {}
}
if (id == null || id.isEmpty()) {
throw new IllegalArgumentException("is empty");
}
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(id);
} catch (Exception e) {
throw new ParsingException("Could not get uploader url");
}
return url;
}
@Nullable
@Override
public String getTextualUploadDate() {
try {
return videoInfo.getObject("publishedTimeText").getString("simpleText");
return getTextFromObject(videoInfo.getObject("publishedTimeText"));
} catch (Exception e) {
// upload date is not always available, e.g. in playlists
return null;
@ -185,15 +194,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
if (videoInfo.getObject("topStandaloneBadge") != null || isPremium()) {
return -1;
}
String viewCount;
if (getStreamType() == StreamType.LIVE_STREAM) {
viewCount = videoInfo.getObject("viewCountText")
.getArray("runs").getObject(0).getString("text");
} else {
viewCount = videoInfo.getObject("viewCountText").getString("simpleText");
}
if (viewCount.equals("Recommended for you")) return -1;
String viewCount = getTextFromObject(videoInfo.getObject("viewCountText"));
return Long.parseLong(Utils.removeNonDigitCharacters(viewCount));
} catch (NumberFormatException e) {
return -1;
} catch (Exception e) {
throw new ParsingException("Could not get view count", e);
}
@ -203,8 +208,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getThumbnailUrl() throws ParsingException {
try {
// TODO: Don't simply get the first item, but look at all thumbnails and their resolution
return videoInfo.getObject("thumbnail").getArray("thumbnails")
String url = videoInfo.getObject("thumbnail").getArray("thumbnails")
.getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file

@ -25,13 +25,11 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -39,6 +37,9 @@ import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData;
@ -50,11 +51,12 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() +
"?gl=" + getExtractorContentCountry().getCountryCode();
final String url = getUrl() + "?pbj=1&gl="
+ getExtractorContentCountry().getCountryCode();
final Response response = downloader.get(url, getExtractorLocalization());
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
}
@Override
@ -72,8 +74,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
public String getName() throws ParsingException {
String name;
try {
name = initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title")
.getArray("runs").getObject(0).getString("text");
name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
} catch (Exception e) {
throw new ParsingException("Could not get Trending name", e);
}
@ -87,17 +88,21 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray firstPageElements = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer").getArray("items");
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonArray itemSectionRenderers = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents");
for (Object ul : firstPageElements) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
for (Object itemSectionRenderer : itemSectionRenderers) {
JsonObject expandedShelfContentsRenderer = ((JsonObject) itemSectionRenderer).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("shelfRenderer").getObject("content")
.getObject("expandedShelfContentsRenderer");
if (expandedShelfContentsRenderer != null) {
for (Object ul : expandedShelfContentsRenderer.getArray("items")) {
final JsonObject videoInfo = ((JsonObject) ul).getObject("videoRenderer");
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View file

@ -35,6 +35,14 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
return instance;
}
/**
* Returns URL to channel from an ID
*
* @param id Channel ID including e.g. 'channel/'
* @param contentFilters
* @param searchFilter
* @return URL to channel
*/
@Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) {
return "https://www.youtube.com/" + id;

229
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View file

@ -5,18 +5,34 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -43,7 +59,9 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@ -160,58 +178,189 @@ public class YoutubeParsingHelper {
}
}
public static boolean isHardcodedClientVersionValid() throws IOException {
try {
final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers).responseBody();
if (response.length() > 50) { // ensure to have a valid response
return true;
}
} catch (ReCaptchaException ignored) {}
return false;
}
/**
* Get the client version from a page
* @param initialData
* @param html The page HTML
* @return
* @throws ParsingException
*/
public static String getClientVersion(JsonObject initialData, String html) throws ParsingException {
if (initialData == null) initialData = getInitialData(html);
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
public static String getClientVersion() throws ParsingException, IOException {
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
// try to get version from initial data first
for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params");
for (Object param: params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("cver")) {
return p.getString("value");
if (isHardcodedClientVersionValid()) {
clientVersion = HARDCODED_CLIENT_VERSION;
return clientVersion;
}
// Try extracting it from YouTube's website otherwise
try {
final String url = "https://www.youtube.com/results?search_query=test";
final String html = getDownloader().get(url).responseBody();
JsonObject initialData = getInitialData(html);
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first
for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("cver")) {
clientVersion = p.getString("value");
return clientVersion;
}
}
}
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last do digits
JsonArray params = s.getArray("params");
for (Object param: params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("client.version")) {
shortClientVersion = p.getString("value");
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last two digits
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("client.version")) {
shortClientVersion = p.getString("value");
}
}
}
}
}
String clientVersion;
String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)"
};
for (String pattern: patterns) {
try {
clientVersion = Parser.matchGroup1(pattern, html);
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
} catch (Exception ignored) {}
}
String contextClientVersion;
String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)"
};
for (String pattern : patterns) {
try {
contextClientVersion = Parser.matchGroup1(pattern, html);
if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
clientVersion = contextClientVersion;
return clientVersion;
}
} catch (Exception ignored) {
}
}
if (shortClientVersion != null) return shortClientVersion;
if (shortClientVersion != null) {
clientVersion = shortClientVersion;
return clientVersion;
}
} catch (Exception ignored) {}
throw new ParsingException("Could not get client version");
}
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
String[] params = internUrl.split("&");
for (String param : params) {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
return null;
}
return url;
}
}
} else if (internUrl.startsWith("http")) {
return internUrl;
}
} else if (navigationEndpoint.getObject("browseEndpoint") != null) {
return "https://www.youtube.com" + navigationEndpoint.getObject("browseEndpoint").getString("canonicalBaseUrl");
} else if (navigationEndpoint.getObject("watchEndpoint") != null) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&amp;list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
}
return null;
}
public static String getTextFromObject(JsonObject textObject, boolean html) {
if (textObject.has("simpleText")) return textObject.getString("simpleText");
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).getObject("navigationEndpoint") != null) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
textBuilder.append("<a href=\"").append(url).append("\">").append(text).append("</a>");
continue;
}
}
textBuilder.append(text);
}
String text = textBuilder.toString();
if (html) {
text = text.replaceAll("\\n", "<br>");
text = text.replaceAll(" ", " &nbsp;");
}
return text;
}
public static String getTextFromObject(JsonObject textObject) {
return getTextFromObject(textObject, false);
}
public static String fixThumbnailUrl(String thumbnailUrl) {
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = thumbnailUrl.substring(2);
}
if (thumbnailUrl.startsWith(HTTP)) {
thumbnailUrl = Utils.replaceHttpWithHttps(thumbnailUrl);
} else if (!thumbnailUrl.startsWith(HTTPS)) {
thumbnailUrl = "https://" + thumbnailUrl;
}
return thumbnailUrl;
}
public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final String response = getDownloader().get(url, headers, localization).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("JSON response is too short");
}
try {
return JsonParser.array().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java Normal file
View file

@ -0,0 +1,24 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
public class YoutubeParsingHelperTest {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void testIsHardcodedClientVersionValid() throws IOException {
assertTrue("Hardcoded client version is not valid anymore",
YoutubeParsingHelper.isHardcodedClientVersionValid());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View file

@ -99,7 +99,7 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCs72iRpTEuwV3y6pdWYLgiw", extractor.getUploaderUrl());
assertEquals("https://www.youtube.com/user/andre0y0you", extractor.getUploaderUrl());
}
@Test

46
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View file

@ -12,10 +12,17 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import java.util.regex.Pattern;
import java.net.URL;
import java.net.URLDecoder;
import java.util.LinkedHashMap;
import java.util.Map;
import static java.util.Arrays.asList;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtractorBaseTest {
@ -47,18 +54,26 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
}
}
assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl());
}
@Test
public void testGetSecondPageUrl() throws Exception {
// check that ctoken, continuation and itct are longer than 5 characters
Pattern pattern = Pattern.compile(
"https:\\/\\/www.youtube.com\\/results\\?search_query=pewdiepie&sp=EgIQAg%253D%253D&gl=GB&pbj=1"
+ "&ctoken=[\\w%]{5,}?&continuation=[\\w%]{5,}?&itct=[\\w]{5,}?"
);
assertTrue(pattern.matcher(extractor.getNextPageUrl()).find());
URL url = new URL(extractor.getNextPageUrl());
assertEquals(url.getHost(), "www.youtube.com");
assertEquals(url.getPath(), "/results");
Map<String, String> queryPairs = new LinkedHashMap<>();
for (String queryPair : url.getQuery().split("&")) {
int index = queryPair.indexOf("=");
queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"),
URLDecoder.decode(queryPair.substring(index + 1), "UTF-8"));
}
assertEquals("pewdiepie", queryPairs.get("search_query"));
assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation"));
assertTrue(queryPairs.get("continuation").length() > 5);
assertTrue(queryPairs.get("itct").length() > 5);
}
@Ignore
@ -77,13 +92,18 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
if (item instanceof ChannelInfoItem) {
ChannelInfoItem channel = (ChannelInfoItem) item;
if (channel.getSubscriberCount() > 5e7) { // the real PewDiePie
if (channel.getSubscriberCount() > 1e8) { // the real PewDiePie
assertEquals("https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw", item.getUrl());
} else {
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
break;
}
}
}
for (InfoItem item : itemsPage.getItems()) {
if (item instanceof ChannelInfoItem) {
assertThat(item.getUrl(), CoreMatchers.startsWith("https://www.youtube.com/channel/"));
}
}
}
@Test

30
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java
View file

@ -10,6 +10,11 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.net.URL;
import java.net.URLDecoder;
import java.util.LinkedHashMap;
import java.util.Map;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -48,13 +53,28 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
@Test
public void testGetUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl());
assertEquals("https://www.youtube.com/results?search_query=pewdiepie&gl=GB", extractor.getUrl());
}
@Test
public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl());
URL url = new URL(extractor.getNextPageUrl());
assertEquals(url.getHost(), "www.youtube.com");
assertEquals(url.getPath(), "/results");
Map<String, String> queryPairs = new LinkedHashMap<>();
for (String queryPair : url.getQuery().split("&")) {
int index = queryPair.indexOf("=");
queryPairs.put(URLDecoder.decode(queryPair.substring(0, index), "UTF-8"),
URLDecoder.decode(queryPair.substring(index + 1), "UTF-8"));
}
assertEquals("pewdiepie", queryPairs.get("search_query"));
assertEquals(queryPairs.get("ctoken"), queryPairs.get("continuation"));
assertTrue(queryPairs.get("continuation").length() > 5);
assertTrue(queryPairs.get("itct").length() > 5);
}
@Test
@ -101,14 +121,12 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
}
}
assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl());
}
@Test
public void testSuggestionNotNull() throws Exception {
public void testSuggestionNotNull() {
//todo write a real test
assertTrue(extractor.getSearchSuggestion() != null);
assertNotNull(extractor.getSearchSuggestion());
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View file

@ -11,11 +11,11 @@ public class YoutubeSearchQHTest {
@Test
public void testRegularValues() throws Exception {
assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://www.youtube.com/results?q=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
}
@Test

42
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View file

@ -10,17 +10,24 @@ import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.List;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -89,7 +96,6 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
assertFalse(extractor.getDescription().getContent().contains("http://smarturl.it/SubscribeAdele?IQi..."));
}
@Test
@ -142,12 +148,12 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
public void testGetAudioStreams() throws ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
@ -169,7 +175,7 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
@ -177,13 +183,13 @@ public class YoutubeStreamExtractorDefaultTest {
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
public void testGetSubtitlesListDefault() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
public void testGetSubtitlesList() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
@ -223,10 +229,6 @@ public class YoutubeStreamExtractorDefaultTest {
assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
assertFalse(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSub..."));
assertFalse(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8..."));
assertFalse(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/..."));
}
}
@ -249,15 +251,11 @@ public class YoutubeStreamExtractorDefaultTest {
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/X7FLCHVXpsA?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/Lqv6G0pDNnw?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/XxaRBPyrnBU?list=PL7..."));
assertFalse(extractor.getDescription().getContent().contains("https://youtu.be/U-9tUEOFKNU?list=PL7..."));
final String description = extractor.getDescription().getContent();
assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
}
}

20
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View file

@ -13,9 +13,12 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -26,7 +29,7 @@ public class YoutubeStreamExtractorLivestreamTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY");
.getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A");
extractor.fetchPage();
}
@ -49,8 +52,7 @@ public class YoutubeStreamExtractorLivestreamTest {
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.baraton/"));
assertFalse(extractor.getDescription().getContent().contains("https://www.instagram.com/nathalie.ba..."));
assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists"));
}
@Test
@ -119,7 +121,7 @@ public class YoutubeStreamExtractorLivestreamTest {
}
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
@ -127,12 +129,12 @@ public class YoutubeStreamExtractorLivestreamTest {
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
public void testGetSubtitlesListDefault() {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
public void testGetSubtitlesList() {
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
}