Cleanup `YoutubeStreamExtractor` and some related classes

* Fixed obvious sonar(lint) warnings
* Abstracted some code (get*Streams)
* Used some new lines to make code better readable
* Chopped down brace-jungle in some methods
* Use StandardCharset (Java 8 4tw)
This commit is contained in:
litetex 2022-05-01 16:38:05 +02:00
parent 52fa2d939a
commit fe30eb43a9
3 changed files with 394 additions and 353 deletions

View file

@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
@ -65,6 +64,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
@ -77,6 +77,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -187,19 +189,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return playerMicroFormatRenderer.getString("uploadDate");
} else if (!playerMicroFormatRenderer.getString("publishDate", EMPTY_STRING).isEmpty()) {
return playerMicroFormatRenderer.getString("publishDate");
} else {
final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
"liveBroadcastDetails");
if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
// an ended live stream
return liveDetails.getString("endTimestamp");
} else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
// a running live stream
return liveDetails.getString("startTimestamp");
} else if (getStreamType() == StreamType.LIVE_STREAM) {
// this should never be reached, but a live stream without upload date is valid
return null;
}
}
final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
"liveBroadcastDetails");
if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
// an ended live stream
return liveDetails.getString("endTimestamp");
} else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
// a running live stream
return liveDetails.getString("startTimestamp");
} else if (getStreamType() == StreamType.LIVE_STREAM) {
// this should never be reached, but a live stream without upload date is valid
return null;
}
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
@ -259,10 +261,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getThumbnailUrl() throws ParsingException {
assertPageFetched();
try {
final JsonArray thumbnails = playerResponse.getObject("videoDetails")
.getObject("thumbnail").getArray("thumbnails");
final JsonArray thumbnails = playerResponse
.getObject("videoDetails")
.getObject("thumbnail")
.getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
final String url = thumbnails
.getObject(thumbnails.size() - 1)
.getString("url");
return fixThumbnailUrl(url);
} catch (final Exception e) {
@ -277,8 +283,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
// Description with more info on links
try {
final String description = getTextFromObject(getVideoSecondaryInfoRenderer()
.getObject("description"), true);
final String description = getTextFromObject(
getVideoSecondaryInfoRenderer().getObject("description"),
true);
if (!isNullOrEmpty(description)) {
return new Description(description, Description.HTML);
}
@ -299,27 +306,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public int getAgeLimit() throws ParsingException {
if (ageLimit == -1) {
ageLimit = NO_AGE_LIMIT;
final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getArray("rows");
for (final Object metadataRow : metadataRows) {
final JsonArray contents = ((JsonObject) metadataRow)
.getObject("metadataRowRenderer").getArray("contents");
for (final Object content : contents) {
final JsonArray runs = ((JsonObject) content).getArray("runs");
for (final Object run : runs) {
final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
if (rowText.contains("Age-restricted")) {
ageLimit = 18;
return ageLimit;
}
}
}
}
if (ageLimit != -1) {
return ageLimit;
}
final boolean ageRestricted = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer")
.getObject("metadataRowContainerRenderer")
.getArray("rows")
.stream()
// Only JsonObjects allowed
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.flatMap(metadataRow -> metadataRow
.getObject("metadataRowRenderer")
.getArray("contents")
.stream()
// Only JsonObjects allowed
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.flatMap(content -> content
.getArray("runs")
.stream()
// Only JsonObjects allowed
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast))
.map(run -> run.getString("text", EMPTY_STRING))
.anyMatch(rowText -> rowText.contains("Age-restricted"));
ageLimit = ageRestricted ? 18 : NO_AGE_LIMIT;
return ageLimit;
}
@ -370,9 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (timestamp == -2) {
// Regex for timestamp was not found
return 0;
} else {
return timestamp;
}
return timestamp;
}
@Override
@ -476,10 +490,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public boolean isUploaderVerified() throws ParsingException {
final JsonArray badges = getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getArray("badges");
return YoutubeParsingHelper.isVerified(badges);
return YoutubeParsingHelper.isVerified(
getVideoSecondaryInfoRenderer()
.getObject("owner")
.getObject("videoOwnerRenderer")
.getArray("badges"));
}
@Nonnull
@ -490,9 +505,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String url = null;
try {
url = getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
url = getVideoSecondaryInfoRenderer()
.getObject("owner")
.getObject("videoOwnerRenderer")
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
} catch (final ParsingException ignored) {
// Age-restricted videos cause a ParsingException here
}
@ -530,8 +549,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// There is no DASH manifest available in the iOS clients and the DASH manifest of the
// Android client doesn't contain all available streams (mainly the WEBM ones)
return getManifestUrl("dash", Arrays.asList(html5StreamingData,
androidStreamingData));
return getManifestUrl(
"dash",
Arrays.asList(html5StreamingData, androidStreamingData));
}
@Nonnull
@ -542,93 +562,91 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
// returned has separated audio and video streams
// Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
return getManifestUrl("hls", Arrays.asList(iosStreamingData, html5StreamingData,
androidStreamingData));
return getManifestUrl(
"hls",
Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData));
}
@Nonnull
private static String getManifestUrl(@Nonnull final String manifestType,
@Nonnull final List<JsonObject> streamingDataObjects) {
final String manifestKey = manifestType + "ManifestUrl";
for (final JsonObject streamingDataObject : streamingDataObjects) {
if (streamingDataObject != null) {
final String manifestKeyValue = streamingDataObject.getString(manifestKey);
if (manifestKeyValue != null) {
return manifestKeyValue;
return streamingDataObjects.stream()
.filter(Objects::nonNull)
.map(streamingDataObject -> streamingDataObject.getString(manifestKey))
.filter(Objects::nonNull)
.findFirst()
.orElse(EMPTY_STRING);
}
@FunctionalInterface
interface StreamTypeStreamBuilderHelper<T extends Stream> {
T buildStream(String url, ItagItem itagItem);
}
/**
* Abstract method for
* {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}.
*
* @param itags A map of Urls + ItagItems
* @param streamBuilder Builds the stream from the provided data
* @param exMsgStreamType Stream type inside the exception message e.g. "video streams"
* @param <T> Type of the stream
* @return
* @throws ExtractionException
*/
private <T extends Stream> List<T> getStreamsByType(
final Map<String, ItagItem> itags,
final StreamTypeStreamBuilderHelper<T> streamBuilder,
final String exMsgStreamType
) throws ExtractionException {
final List<T> streams = new ArrayList<>();
try {
for (final Map.Entry<String, ItagItem> entry : itags.entrySet()) {
final String url = tryDecryptUrl(entry.getKey(), getId());
final T stream = streamBuilder.buildStream(url, entry.getValue());
if (!Stream.containSimilarStream(stream, streams)) {
streams.add(stream);
}
}
} catch (final Exception e) {
throw new ParsingException("Could not get " + exMsgStreamType, e);
}
return EMPTY_STRING;
return streams;
}
@Override
public List<AudioStream> getAudioStreams() throws ExtractionException {
assertPageFetched();
final List<AudioStream> audioStreams = new ArrayList<>();
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.AUDIO).entrySet()) {
final ItagItem itag = entry.getValue();
final String url = tryDecryption(entry.getKey(), getId());
final AudioStream audioStream = new AudioStream(url, itag);
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
}
} catch (final Exception e) {
throw new ParsingException("Could not get audio streams", e);
}
return audioStreams;
return getStreamsByType(
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO),
AudioStream::new,
"audio streams"
);
}
@Override
public List<VideoStream> getVideoStreams() throws ExtractionException {
assertPageFetched();
final List<VideoStream> videoStreams = new ArrayList<>();
try {
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
ItagItem.ItagType.VIDEO).entrySet()) {
final ItagItem itag = entry.getValue();
final String url = tryDecryption(entry.getKey(), getId());
final VideoStream videoStream = new VideoStream(url, false, itag);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
}
} catch (final Exception e) {
throw new ParsingException("Could not get video streams", e);
}
return videoStreams;
return getStreamsByType(
getItags(FORMATS, ItagItem.ItagType.VIDEO),
(url, itag) -> new VideoStream(url, false, itag),
"video streams"
);
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
assertPageFetched();
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
final ItagItem itag = entry.getValue();
final String url = tryDecryption(entry.getKey(), getId());
final VideoStream videoStream = new VideoStream(url, true, itag);
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
videoOnlyStreams.add(videoStream);
}
}
} catch (final Exception e) {
throw new ParsingException("Could not get video only streams", e);
}
return videoOnlyStreams;
return getStreamsByType(
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY),
(url, itag) -> new VideoStream(url, true, itag),
"video only streams"
);
}
/**
@ -636,7 +654,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
* always needed.
* This way a breaking change from YouTube does not result in a broken extractor.
*/
private String tryDecryption(final String url, final String videoId) {
private String tryDecryptUrl(final String url, final String videoId) {
try {
return YoutubeThrottlingDecrypter.apply(url, videoId);
} catch (final ParsingException e) {
@ -713,25 +731,33 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final JsonArray results = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
.getObject("secondaryResults").getArray("results");
final JsonArray results = nextResponse
.getObject("contents")
.getObject("twoColumnWatchNextResults")
.getObject("secondaryResults")
.getObject("secondaryResults")
.getArray("results");
final TimeAgoParser timeAgoParser = getTimeAgoParser();
results.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(result -> {
if (result.has("compactVideoRenderer")) {
return new YoutubeStreamInfoItemExtractor(
result.getObject("compactVideoRenderer"), timeAgoParser);
} else if (result.has("compactRadioRenderer")) {
return new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactRadioRenderer"));
} else if (result.has("compactPlaylistRenderer")) {
return new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactPlaylistRenderer"));
}
return null;
})
.filter(Objects::nonNull)
.forEach(collector::commit);
for (final Object resultObject : results) {
final JsonObject result = (JsonObject) resultObject;
if (result.has("compactVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
result.getObject("compactVideoRenderer"), timeAgoParser));
} else if (result.has("compactRadioRenderer")) {
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactRadioRenderer")));
} else if (result.has("compactPlaylistRenderer")) {
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
result.getObject("compactPlaylistRenderer")));
}
}
return collector;
} catch (final Exception e) {
throw new ParsingException("Could not get related videos", e);
@ -777,9 +803,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
initStsFromPlayerJsIfNeeded();
final String videoId = getId();
final Localization localization = getExtractorLocalization();
@ -833,13 +857,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
nextResponse = getJsonPostResponse(NEXT, body, localization);
if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
@ -863,53 +887,56 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final JsonObject playabilityStatus)
throws ParsingException {
String status = playabilityStatus.getString("status");
if (status == null || status.equalsIgnoreCase("ok")) {
return;
}
// If status exist, and is not "OK", throw the specific exception based on error message
// or a ContentNotAvailableException with the reason text if it's an unknown reason.
if (status != null && !status.equalsIgnoreCase("ok")) {
final JsonObject newPlayabilityStatus
= youtubePlayerResponse.getObject("playabilityStatus");
status = newPlayabilityStatus.getString("status");
final String reason = newPlayabilityStatus.getString("reason");
final JsonObject newPlayabilityStatus =
youtubePlayerResponse.getObject("playabilityStatus");
status = newPlayabilityStatus.getString("status");
final String reason = newPlayabilityStatus.getString("reason");
if (status.equalsIgnoreCase("login_required")) {
if (reason == null) {
final String message = newPlayabilityStatus.getArray("messages").getString(0);
if (message != null && message.contains("private")) {
throw new PrivateContentException("This video is private.");
}
} else if (reason.contains("age")) {
// No streams can be fetched, therefore throw an AgeRestrictedContentException
// explicitly.
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched.");
if (status.equalsIgnoreCase("login_required")) {
if (reason == null) {
final String message = newPlayabilityStatus.getArray("messages").getString(0);
if (message != null && message.contains("private")) {
throw new PrivateContentException("This video is private.");
}
} else if (reason.contains("age")) {
// No streams can be fetched, therefore throw an AgeRestrictedContentException
// explicitly.
throw new AgeRestrictedContentException(
"This age-restricted video cannot be watched.");
}
if (status.equalsIgnoreCase("unplayable") && reason != null) {
if (reason.contains("Music Premium")) {
throw new YoutubeMusicPremiumContentException();
}
if (reason.contains("payment")) {
throw new PaidContentException("This video is a paid video");
}
if (reason.contains("members-only")) {
throw new PaidContentException("This video is only available"
+ " for members of the channel of this video");
}
if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
.getObject("subreason"));
if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
throw new GeographicRestrictionException(
"This video is not available in client's country.");
}
}
}
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
}
if (status.equalsIgnoreCase("unplayable") && reason != null) {
if (reason.contains("Music Premium")) {
throw new YoutubeMusicPremiumContentException();
}
if (reason.contains("payment")) {
throw new PaidContentException("This video is a paid video");
}
if (reason.contains("members-only")) {
throw new PaidContentException("This video is only available"
+ " for members of the channel of this video");
}
if (reason.contains("unavailable")) {
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
.getObject("errorScreen")
.getObject("playerErrorMessageRenderer")
.getObject("subreason"));
if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
throw new GeographicRestrictionException(
"This video is not available in client's country.");
}
}
}
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
}
/**
@ -921,14 +948,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final String videoId)
throws IOException, ExtractionException {
androidCpn = generateContentPlaybackNonce();
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CPN, androidCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
final byte[] mobileBody = JsonWriter.string(
prepareAndroidMobileJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CPN, androidCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
mobileBody, localization, "&t=" + generateTParameter()
@ -952,14 +979,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final String videoId)
throws IOException, ExtractionException {
iosCpn = generateContentPlaybackNonce();
final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder(
localization, contentCountry)
final byte[] mobileBody = JsonWriter.string(
prepareIosMobileJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CPN, iosCpn)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(UTF_8);
.getBytes(StandardCharsets.UTF_8);
final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER,
mobileBody, localization, "&t=" + generateTParameter()
@ -987,9 +1014,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final Localization localization,
@Nonnull final String videoId)
throws IOException, ExtractionException {
if (sts == null) {
getStsFromPlayerJs();
}
initStsFromPlayerJsIfNeeded();
// Because a cpn is unique to each request, we need to generate it again
html5Cpn = generateContentPlaybackNonce();
@ -1072,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return cachedDeobfuscationCode;
}
private static void getStsFromPlayerJs() throws ParsingException {
private static void initStsFromPlayerJsIfNeeded() throws ParsingException {
if (!isNullOrEmpty(sts)) {
return;
}
@ -1134,31 +1159,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return theVideoPrimaryInfoRenderer;
}
@Nonnull
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
if (videoSecondaryInfoRenderer != null) {
return videoSecondaryInfoRenderer;
}
final JsonArray contents = nextResponse.getObject("contents")
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
.getArray("contents");
videoSecondaryInfoRenderer = nextResponse
.getObject("contents")
.getObject("twoColumnWatchNextResults")
.getObject("results")
.getObject("results")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(content -> content.has("videoSecondaryInfoRenderer"))
.map(content -> content.getObject("videoSecondaryInfoRenderer"))
.findFirst()
.orElseThrow(
() -> new ParsingException("Could not find videoSecondaryInfoRenderer"));
JsonObject theVideoSecondaryInfoRenderer = null;
for (final Object content : contents) {
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
theVideoSecondaryInfoRenderer = ((JsonObject) content)
.getObject("videoSecondaryInfoRenderer");
break;
}
}
if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) {
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
}
videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
return theVideoSecondaryInfoRenderer;
return videoSecondaryInfoRenderer;
}
@Nonnull
@ -1194,83 +1216,82 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted,
@Nonnull final String contentPlaybackNonce) {
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
if (streamingData != null && streamingData.has(streamingDataKey)) {
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
final JsonObject formatData = formats.getObject(i);
final int itag = formatData.getInt("itag");
if (ItagItem.isSupported(itag)) {
try {
final ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
}
final String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url") + "&cpn="
+ contentPlaybackNonce;
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(
cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
final String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
"-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
"-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
"-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
"-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
}
} catch (final UnsupportedEncodingException | ParsingException ignored) {
}
}
}
if (streamingData == null || !streamingData.has(streamingDataKey)) {
return Collections.emptyMap();
}
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i < formats.size(); i++) {
final JsonObject formatData = formats.getObject(i);
final int itag = formatData.getInt("itag");
if (!ItagItem.isSupported(itag)) {
continue;
}
try {
final ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType != itagTypeWanted) {
continue;
}
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) {
continue;
}
final String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(
cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ deobfuscateSignature(cipher.get("s"));
}
final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
final String codec = mimeType.contains("codecs")
? mimeType.split("\"")[1]
: EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
} catch (final UnsupportedEncodingException | ParsingException ignored) {
}
}
return urlAndItagsFromStreamingDataObject;
}
@Nonnull
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
final JsonObject storyboards = playerResponse.getObject("storyboards");
final JsonObject storyboardsRenderer;
if (storyboards.has("playerLiveStoryboardSpecRenderer")) {
storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer");
} else {
storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer");
}
final JsonObject storyboardsRenderer = storyboards.getObject(
storyboards.has("playerLiveStoryboardSpecRenderer")
? "playerLiveStoryboardSpecRenderer"
: "playerStoryboardSpecRenderer"
);
if (storyboardsRenderer == null) {
return Collections.emptyList();
@ -1283,15 +1304,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String[] spec = storyboardsRendererSpec.split("\\|");
final String url = spec[0];
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
final List<Frameset> result = new ArrayList<>(spec.length - 1);
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8 || Integer.parseInt(parts[5]) == 0) {
continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
final int frameHeight = Integer.parseInt(parts[1]);
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
@ -1310,15 +1329,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
result.add(new Frameset(
urls,
frameWidth,
frameHeight,
/*frameWidth=*/Integer.parseInt(parts[0]),
/*frameHeight=*/Integer.parseInt(parts[1]),
totalCount,
Integer.parseInt(parts[5]),
/*durationPerFrame=*/Integer.parseInt(parts[5]),
framesPerPageX,
framesPerPageY
));
}
result.trimToSize();
return result;
} catch (final Exception e) {
throw new ExtractionException("Could not get frames", e);
@ -1328,8 +1346,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public Privacy getPrivacy() {
final boolean isUnlisted = playerMicroFormatRenderer.getBoolean("isUnlisted");
return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
return playerMicroFormatRenderer.getBoolean("isUnlisted")
? Privacy.UNLISTED
: Privacy.PUBLIC;
}
@Nonnull
@ -1342,14 +1361,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getLicence() throws ParsingException {
final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getObject("metadataRowContainer")
.getObject("metadataRowContainerRenderer")
.getArray("rows")
.getObject(0).getObject("metadataRowRenderer");
.getObject(0)
.getObject("metadataRowRenderer");
final JsonArray contents = metadataRowRenderer.getArray("contents");
final String license = getTextFromObject(contents.getObject(0));
return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer
.getObject("title"))) ? license : "YouTube licence";
return license != null
&& "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title")))
? license
: "YouTube licence";
}
@Override
@ -1367,63 +1390,73 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() throws ParsingException {
final ArrayList<StreamSegment> segments = new ArrayList<>();
if (nextResponse.has("engagementPanels")) {
final JsonArray panels = nextResponse.getArray("engagementPanels");
JsonArray segmentsArray = null;
// Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) {
final String panelIdentifier = panels.getObject(i)
if (!nextResponse.has("engagementPanels")) {
return Collections.emptyList();
}
final JsonArray segmentsArray = nextResponse.getArray("engagementPanels")
.stream()
// Check if object is a JsonObject
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
// Check if the panel is the correct one
.filter(panel -> "engagement-panel-macro-markers-description-chapters".equals(
panel
.getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier")))
// Extract the data
.map(panel -> panel
.getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier");
// panelIdentifier might be null if the panel has something to do with ads
// See https://github.com/TeamNewPipe/NewPipe/issues/7792#issuecomment-1030900188
if ("engagement-panel-macro-markers-description-chapters".equals(panelIdentifier)) {
segmentsArray = panels.getObject(i)
.getObject("engagementPanelSectionListRenderer").getObject("content")
.getObject("macroMarkersListRenderer").getArray("contents");
break;
}
.getObject("content")
.getObject("macroMarkersListRenderer")
.getArray("contents"))
.findFirst()
.orElse(null);
// If no data was found exit
if (segmentsArray == null) {
return Collections.emptyList();
}
final long duration = getLength();
final List<StreamSegment> segments = new ArrayList<>();
for (final JsonObject segmentJson : segmentsArray.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(object -> object.getObject("macroMarkersListItemRenderer"))
.collect(Collectors.toList())
) {
final int startTimeSeconds = segmentJson.getObject("onTap")
.getObject("watchEndpoint").getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) {
throw new ParsingException("Could not get stream segment start time.");
}
if (startTimeSeconds > duration) {
break;
}
if (segmentsArray != null) {
final long duration = getLength();
for (final Object object : segmentsArray) {
final JsonObject segmentJson = ((JsonObject) object)
.getObject("macroMarkersListItemRenderer");
final String title = getTextFromObject(segmentJson.getObject("title"));
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get stream segment title.");
}
final int startTimeSeconds = segmentJson.getObject("onTap")
.getObject("watchEndpoint").getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) {
throw new ParsingException("Could not get stream segment start time.");
}
if (startTimeSeconds > duration) {
break;
}
final String title = getTextFromObject(segmentJson.getObject("title"));
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get stream segment title.");
}
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson.getObject("thumbnail")
.getArray("thumbnails");
if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the
// last position
final String url = previewsArray
.getObject(previewsArray.size() - 1).getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url));
}
}
segments.add(segment);
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson
.getObject("thumbnail")
.getArray("thumbnails");
if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the last position
final String url = previewsArray
.getObject(previewsArray.size() - 1)
.getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url));
}
}
segments.add(segment);
}
return segments;
}
@ -1431,9 +1464,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
nextResponse.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"));
return YoutubeParsingHelper.getMetaInfo(nextResponse
.getObject("contents")
.getObject("twoColumnWatchNextResults")
.getObject("results")
.getObject("results")
.getArray("contents"));
}
/**

View file

@ -1,19 +1,19 @@
package org.schabi.newpipe.extractor.stream;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Creates a stream object from url, format and optional torrent url
*/
public abstract class Stream implements Serializable {
private final MediaFormat mediaFormat;
public final String url;
public final String torrentUrl;
private final String url;
private final String torrentUrl;
/**
* @deprecated Use {@link #getFormat()} or {@link #getFormatId()}

View file

@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
@ -18,10 +16,17 @@ import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public final class Utils {
public static final String HTTP = "http://";
public static final String HTTPS = "https://";
/**
* @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8}
*/
@Deprecated
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");