Merge pull request #782 from litetex/cleanup-yt-stream-extractor
Cleanup of ``YoutubeStreamExtractor`` and some related classes
This commit is contained in:
litetex
GitHub
commit
5db4d1faf3
3 changed files with 394 additions and 353 deletions
|
|
@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
|
|||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
|
|
@ -66,6 +65,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDate;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
|
@ -77,6 +77,8 @@ import java.util.List;
|
|||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
|
|
@ -187,7 +189,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return playerMicroFormatRenderer.getString("uploadDate");
|
||||
} else if (!playerMicroFormatRenderer.getString("publishDate", EMPTY_STRING).isEmpty()) {
|
||||
return playerMicroFormatRenderer.getString("publishDate");
|
||||
} else {
|
||||
}
|
||||
|
||||
final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
|
||||
"liveBroadcastDetails");
|
||||
if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
|
||||
|
|
@ -200,7 +203,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
// this should never be reached, but a live stream without upload date is valid
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
|
||||
.startsWith("Premiered")) {
|
||||
|
|
@ -259,10 +261,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
public String getThumbnailUrl() throws ParsingException {
|
||||
assertPageFetched();
|
||||
try {
|
||||
final JsonArray thumbnails = playerResponse.getObject("videoDetails")
|
||||
.getObject("thumbnail").getArray("thumbnails");
|
||||
final JsonArray thumbnails = playerResponse
|
||||
.getObject("videoDetails")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
||||
final String url = thumbnails
|
||||
.getObject(thumbnails.size() - 1)
|
||||
.getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
} catch (final Exception e) {
|
||||
|
|
@ -277,8 +283,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
assertPageFetched();
|
||||
// Description with more info on links
|
||||
try {
|
||||
final String description = getTextFromObject(getVideoSecondaryInfoRenderer()
|
||||
.getObject("description"), true);
|
||||
final String description = getTextFromObject(
|
||||
getVideoSecondaryInfoRenderer().getObject("description"),
|
||||
true);
|
||||
if (!isNullOrEmpty(description)) {
|
||||
return new Description(description, Description.HTML);
|
||||
}
|
||||
|
|
@ -299,27 +306,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public int getAgeLimit() throws ParsingException {
|
||||
if (ageLimit == -1) {
|
||||
ageLimit = NO_AGE_LIMIT;
|
||||
|
||||
final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
|
||||
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
|
||||
.getArray("rows");
|
||||
for (final Object metadataRow : metadataRows) {
|
||||
final JsonArray contents = ((JsonObject) metadataRow)
|
||||
.getObject("metadataRowRenderer").getArray("contents");
|
||||
for (final Object content : contents) {
|
||||
final JsonArray runs = ((JsonObject) content).getArray("runs");
|
||||
for (final Object run : runs) {
|
||||
final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
|
||||
if (rowText.contains("Age-restricted")) {
|
||||
ageLimit = 18;
|
||||
if (ageLimit != -1) {
|
||||
return ageLimit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final boolean ageRestricted = getVideoSecondaryInfoRenderer()
|
||||
.getObject("metadataRowContainer")
|
||||
.getObject("metadataRowContainerRenderer")
|
||||
.getArray("rows")
|
||||
.stream()
|
||||
// Only JsonObjects allowed
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.flatMap(metadataRow -> metadataRow
|
||||
.getObject("metadataRowRenderer")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
// Only JsonObjects allowed
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast))
|
||||
.flatMap(content -> content
|
||||
.getArray("runs")
|
||||
.stream()
|
||||
// Only JsonObjects allowed
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast))
|
||||
.map(run -> run.getString("text", EMPTY_STRING))
|
||||
.anyMatch(rowText -> rowText.contains("Age-restricted"));
|
||||
|
||||
ageLimit = ageRestricted ? 18 : NO_AGE_LIMIT;
|
||||
return ageLimit;
|
||||
}
|
||||
|
||||
|
|
@ -370,9 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
if (timestamp == -2) {
|
||||
// Regex for timestamp was not found
|
||||
return 0;
|
||||
} else {
|
||||
return timestamp;
|
||||
}
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -476,10 +490,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public boolean isUploaderVerified() throws ParsingException {
|
||||
final JsonArray badges = getVideoSecondaryInfoRenderer().getObject("owner")
|
||||
.getObject("videoOwnerRenderer").getArray("badges");
|
||||
|
||||
return YoutubeParsingHelper.isVerified(badges);
|
||||
return YoutubeParsingHelper.isVerified(
|
||||
getVideoSecondaryInfoRenderer()
|
||||
.getObject("owner")
|
||||
.getObject("videoOwnerRenderer")
|
||||
.getArray("badges"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -490,9 +505,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
String url = null;
|
||||
|
||||
try {
|
||||
url = getVideoSecondaryInfoRenderer().getObject("owner")
|
||||
.getObject("videoOwnerRenderer").getObject("thumbnail")
|
||||
.getArray("thumbnails").getObject(0).getString("url");
|
||||
url = getVideoSecondaryInfoRenderer()
|
||||
.getObject("owner")
|
||||
.getObject("videoOwnerRenderer")
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails")
|
||||
.getObject(0)
|
||||
.getString("url");
|
||||
} catch (final ParsingException ignored) {
|
||||
// Age-restricted videos cause a ParsingException here
|
||||
}
|
||||
|
|
@ -530,8 +549,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
// There is no DASH manifest available in the iOS clients and the DASH manifest of the
|
||||
// Android client doesn't contain all available streams (mainly the WEBM ones)
|
||||
return getManifestUrl("dash", Arrays.asList(html5StreamingData,
|
||||
androidStreamingData));
|
||||
return getManifestUrl(
|
||||
"dash",
|
||||
Arrays.asList(html5StreamingData, androidStreamingData));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -542,93 +562,91 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
// Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
|
||||
// returned has separated audio and video streams
|
||||
// Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
|
||||
return getManifestUrl("hls", Arrays.asList(iosStreamingData, html5StreamingData,
|
||||
androidStreamingData));
|
||||
return getManifestUrl(
|
||||
"hls",
|
||||
Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private static String getManifestUrl(@Nonnull final String manifestType,
|
||||
@Nonnull final List<JsonObject> streamingDataObjects) {
|
||||
final String manifestKey = manifestType + "ManifestUrl";
|
||||
for (final JsonObject streamingDataObject : streamingDataObjects) {
|
||||
if (streamingDataObject != null) {
|
||||
final String manifestKeyValue = streamingDataObject.getString(manifestKey);
|
||||
if (manifestKeyValue != null) {
|
||||
return manifestKeyValue;
|
||||
}
|
||||
}
|
||||
|
||||
return streamingDataObjects.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(streamingDataObject -> streamingDataObject.getString(manifestKey))
|
||||
.filter(Objects::nonNull)
|
||||
.findFirst()
|
||||
.orElse(EMPTY_STRING);
|
||||
}
|
||||
|
||||
return EMPTY_STRING;
|
||||
@FunctionalInterface
|
||||
interface StreamTypeStreamBuilderHelper<T extends Stream> {
|
||||
T buildStream(String url, ItagItem itagItem);
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract method for
|
||||
* {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}.
|
||||
*
|
||||
* @param itags A map of Urls + ItagItems
|
||||
* @param streamBuilder Builds the stream from the provided data
|
||||
* @param exMsgStreamType Stream type inside the exception message e.g. "video streams"
|
||||
* @param <T> Type of the stream
|
||||
* @return
|
||||
* @throws ExtractionException
|
||||
*/
|
||||
private <T extends Stream> List<T> getStreamsByType(
|
||||
final Map<String, ItagItem> itags,
|
||||
final StreamTypeStreamBuilderHelper<T> streamBuilder,
|
||||
final String exMsgStreamType
|
||||
) throws ExtractionException {
|
||||
final List<T> streams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : itags.entrySet()) {
|
||||
final String url = tryDecryptUrl(entry.getKey(), getId());
|
||||
|
||||
final T stream = streamBuilder.buildStream(url, entry.getValue());
|
||||
if (!Stream.containSimilarStream(stream, streams)) {
|
||||
streams.add(stream);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get " + exMsgStreamType, e);
|
||||
}
|
||||
|
||||
return streams;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioStream> getAudioStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<AudioStream> audioStreams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
|
||||
ItagItem.ItagType.AUDIO).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final AudioStream audioStream = new AudioStream(url, itag);
|
||||
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||
audioStreams.add(audioStream);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get audio streams", e);
|
||||
}
|
||||
|
||||
return audioStreams;
|
||||
return getStreamsByType(
|
||||
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO),
|
||||
AudioStream::new,
|
||||
"audio streams"
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoStreams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
|
||||
ItagItem.ItagType.VIDEO).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final VideoStream videoStream = new VideoStream(url, false, itag);
|
||||
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||
videoStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video streams", e);
|
||||
}
|
||||
|
||||
return videoStreams;
|
||||
return getStreamsByType(
|
||||
getItags(FORMATS, ItagItem.ItagType.VIDEO),
|
||||
(url, itag) -> new VideoStream(url, false, itag),
|
||||
"video streams"
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
|
||||
assertPageFetched();
|
||||
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
|
||||
|
||||
try {
|
||||
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
|
||||
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
|
||||
final ItagItem itag = entry.getValue();
|
||||
final String url = tryDecryption(entry.getKey(), getId());
|
||||
|
||||
final VideoStream videoStream = new VideoStream(url, true, itag);
|
||||
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
|
||||
videoOnlyStreams.add(videoStream);
|
||||
}
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get video only streams", e);
|
||||
}
|
||||
|
||||
return videoOnlyStreams;
|
||||
return getStreamsByType(
|
||||
getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY),
|
||||
(url, itag) -> new VideoStream(url, true, itag),
|
||||
"video only streams"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -636,7 +654,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
* always needed.
|
||||
* This way a breaking change from YouTube does not result in a broken extractor.
|
||||
*/
|
||||
private String tryDecryption(final String url, final String videoId) {
|
||||
private String tryDecryptUrl(final String url, final String videoId) {
|
||||
try {
|
||||
return YoutubeThrottlingDecrypter.apply(url, videoId);
|
||||
} catch (final ParsingException e) {
|
||||
|
|
@ -713,25 +731,33 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
try {
|
||||
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
|
||||
|
||||
final JsonArray results = nextResponse.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults").getObject("secondaryResults")
|
||||
.getObject("secondaryResults").getArray("results");
|
||||
final JsonArray results = nextResponse
|
||||
.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults")
|
||||
.getObject("secondaryResults")
|
||||
.getObject("secondaryResults")
|
||||
.getArray("results");
|
||||
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
for (final Object resultObject : results) {
|
||||
final JsonObject result = (JsonObject) resultObject;
|
||||
results.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(result -> {
|
||||
if (result.has("compactVideoRenderer")) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(
|
||||
result.getObject("compactVideoRenderer"), timeAgoParser));
|
||||
return new YoutubeStreamInfoItemExtractor(
|
||||
result.getObject("compactVideoRenderer"), timeAgoParser);
|
||||
} else if (result.has("compactRadioRenderer")) {
|
||||
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactRadioRenderer")));
|
||||
return new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactRadioRenderer"));
|
||||
} else if (result.has("compactPlaylistRenderer")) {
|
||||
collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactPlaylistRenderer")));
|
||||
}
|
||||
return new YoutubeMixOrPlaylistInfoItemExtractor(
|
||||
result.getObject("compactPlaylistRenderer"));
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter(Objects::nonNull)
|
||||
.forEach(collector::commit);
|
||||
|
||||
return collector;
|
||||
} catch (final Exception e) {
|
||||
throw new ParsingException("Could not get related videos", e);
|
||||
|
|
@ -777,9 +803,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader)
|
||||
throws IOException, ExtractionException {
|
||||
if (sts == null) {
|
||||
getStsFromPlayerJs();
|
||||
}
|
||||
initStsFromPlayerJsIfNeeded();
|
||||
|
||||
final String videoId = getId();
|
||||
final Localization localization = getExtractorLocalization();
|
||||
|
|
@ -833,13 +857,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat")
|
||||
.getObject("playerMicroformatRenderer");
|
||||
|
||||
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
|
||||
contentCountry)
|
||||
final byte[] body = JsonWriter.string(
|
||||
prepareDesktopJsonBuilder(localization, contentCountry)
|
||||
.value(VIDEO_ID, videoId)
|
||||
.value(CONTENT_CHECK_OK, true)
|
||||
.value(RACY_CHECK_OK, true)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
nextResponse = getJsonPostResponse(NEXT, body, localization);
|
||||
|
||||
if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
|
||||
|
|
@ -863,11 +887,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull final JsonObject playabilityStatus)
|
||||
throws ParsingException {
|
||||
String status = playabilityStatus.getString("status");
|
||||
if (status == null || status.equalsIgnoreCase("ok")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If status exist, and is not "OK", throw the specific exception based on error message
|
||||
// or a ContentNotAvailableException with the reason text if it's an unknown reason.
|
||||
if (status != null && !status.equalsIgnoreCase("ok")) {
|
||||
final JsonObject newPlayabilityStatus
|
||||
= youtubePlayerResponse.getObject("playabilityStatus");
|
||||
final JsonObject newPlayabilityStatus =
|
||||
youtubePlayerResponse.getObject("playabilityStatus");
|
||||
status = newPlayabilityStatus.getString("status");
|
||||
final String reason = newPlayabilityStatus.getString("reason");
|
||||
|
||||
|
|
@ -899,7 +926,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
if (reason.contains("unavailable")) {
|
||||
final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
|
||||
.getObject("errorScreen").getObject("playerErrorMessageRenderer")
|
||||
.getObject("errorScreen")
|
||||
.getObject("playerErrorMessageRenderer")
|
||||
.getObject("subreason"));
|
||||
if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
|
||||
throw new GeographicRestrictionException(
|
||||
|
|
@ -910,7 +938,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON
|
||||
|
|
@ -921,14 +948,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull final String videoId)
|
||||
throws IOException, ExtractionException {
|
||||
androidCpn = generateContentPlaybackNonce();
|
||||
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
|
||||
localization, contentCountry)
|
||||
final byte[] mobileBody = JsonWriter.string(
|
||||
prepareAndroidMobileJsonBuilder(localization, contentCountry)
|
||||
.value(VIDEO_ID, videoId)
|
||||
.value(CPN, androidCpn)
|
||||
.value(CONTENT_CHECK_OK, true)
|
||||
.value(RACY_CHECK_OK, true)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
|
||||
mobileBody, localization, "&t=" + generateTParameter()
|
||||
|
|
@ -952,14 +979,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull final String videoId)
|
||||
throws IOException, ExtractionException {
|
||||
iosCpn = generateContentPlaybackNonce();
|
||||
final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder(
|
||||
localization, contentCountry)
|
||||
final byte[] mobileBody = JsonWriter.string(
|
||||
prepareIosMobileJsonBuilder(localization, contentCountry)
|
||||
.value(VIDEO_ID, videoId)
|
||||
.value(CPN, iosCpn)
|
||||
.value(CONTENT_CHECK_OK, true)
|
||||
.value(RACY_CHECK_OK, true)
|
||||
.done())
|
||||
.getBytes(UTF_8);
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
|
||||
final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER,
|
||||
mobileBody, localization, "&t=" + generateTParameter()
|
||||
|
|
@ -987,9 +1014,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull final Localization localization,
|
||||
@Nonnull final String videoId)
|
||||
throws IOException, ExtractionException {
|
||||
if (sts == null) {
|
||||
getStsFromPlayerJs();
|
||||
}
|
||||
initStsFromPlayerJsIfNeeded();
|
||||
|
||||
// Because a cpn is unique to each request, we need to generate it again
|
||||
html5Cpn = generateContentPlaybackNonce();
|
||||
|
|
@ -1072,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return cachedDeobfuscationCode;
|
||||
}
|
||||
|
||||
private static void getStsFromPlayerJs() throws ParsingException {
|
||||
private static void initStsFromPlayerJsIfNeeded() throws ParsingException {
|
||||
if (!isNullOrEmpty(sts)) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -1134,31 +1159,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
return theVideoPrimaryInfoRenderer;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
|
||||
if (videoSecondaryInfoRenderer != null) {
|
||||
return videoSecondaryInfoRenderer;
|
||||
}
|
||||
|
||||
final JsonArray contents = nextResponse.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
|
||||
.getArray("contents");
|
||||
videoSecondaryInfoRenderer = nextResponse
|
||||
.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults")
|
||||
.getObject("results")
|
||||
.getObject("results")
|
||||
.getArray("contents")
|
||||
.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.filter(content -> content.has("videoSecondaryInfoRenderer"))
|
||||
.map(content -> content.getObject("videoSecondaryInfoRenderer"))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new ParsingException("Could not find videoSecondaryInfoRenderer"));
|
||||
|
||||
JsonObject theVideoSecondaryInfoRenderer = null;
|
||||
|
||||
for (final Object content : contents) {
|
||||
if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
|
||||
theVideoSecondaryInfoRenderer = ((JsonObject) content)
|
||||
.getObject("videoSecondaryInfoRenderer");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) {
|
||||
throw new ParsingException("Could not find videoSecondaryInfoRenderer");
|
||||
}
|
||||
|
||||
videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
|
||||
return theVideoSecondaryInfoRenderer;
|
||||
return videoSecondaryInfoRenderer;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -1194,29 +1216,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull final String streamingDataKey,
|
||||
@Nonnull final ItagItem.ItagType itagTypeWanted,
|
||||
@Nonnull final String contentPlaybackNonce) {
|
||||
if (streamingData == null || !streamingData.has(streamingDataKey)) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
|
||||
if (streamingData != null && streamingData.has(streamingDataKey)) {
|
||||
final JsonArray formats = streamingData.getArray(streamingDataKey);
|
||||
for (int i = 0; i != formats.size(); ++i) {
|
||||
for (int i = 0; i < formats.size(); i++) {
|
||||
final JsonObject formatData = formats.getObject(i);
|
||||
final int itag = formatData.getInt("itag");
|
||||
|
||||
if (ItagItem.isSupported(itag)) {
|
||||
if (!ItagItem.isSupported(itag)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
final ItagItem itagItem = ItagItem.getItag(itag);
|
||||
if (itagItem.itagType == itagTypeWanted) {
|
||||
if (itagItem.itagType != itagTypeWanted) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ignore streams that are delivered using YouTube's OTF format,
|
||||
// as those only work with DASH and not with progressive HTTP.
|
||||
if (formatData.getString("type", EMPTY_STRING)
|
||||
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
|
||||
if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final String streamUrl;
|
||||
if (formatData.has("url")) {
|
||||
streamUrl = formatData.getString("url") + "&cpn="
|
||||
+ contentPlaybackNonce;
|
||||
streamUrl = formatData.getString("url");
|
||||
} else {
|
||||
// This url has an obfuscated signature
|
||||
final String cipherString = formatData.has("cipher")
|
||||
|
|
@ -1232,45 +1260,38 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
final JsonObject indexRange = formatData.getObject("indexRange");
|
||||
final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
|
||||
final String codec = mimeType.contains("codecs")
|
||||
? mimeType.split("\"")[1] : EMPTY_STRING;
|
||||
? mimeType.split("\"")[1]
|
||||
: EMPTY_STRING;
|
||||
|
||||
itagItem.setBitrate(formatData.getInt("bitrate"));
|
||||
itagItem.setWidth(formatData.getInt("width"));
|
||||
itagItem.setHeight(formatData.getInt("height"));
|
||||
itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
|
||||
"-1")));
|
||||
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
|
||||
"-1")));
|
||||
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
|
||||
"-1")));
|
||||
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
|
||||
"-1")));
|
||||
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
|
||||
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
|
||||
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
|
||||
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
|
||||
itagItem.fps = formatData.getInt("fps");
|
||||
itagItem.setQuality(formatData.getString("quality"));
|
||||
itagItem.setCodec(codec);
|
||||
|
||||
urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
|
||||
}
|
||||
} catch (final UnsupportedEncodingException | ParsingException ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return urlAndItagsFromStreamingDataObject;
|
||||
}
|
||||
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<Frameset> getFrames() throws ExtractionException {
|
||||
try {
|
||||
final JsonObject storyboards = playerResponse.getObject("storyboards");
|
||||
final JsonObject storyboardsRenderer;
|
||||
if (storyboards.has("playerLiveStoryboardSpecRenderer")) {
|
||||
storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer");
|
||||
} else {
|
||||
storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer");
|
||||
}
|
||||
final JsonObject storyboardsRenderer = storyboards.getObject(
|
||||
storyboards.has("playerLiveStoryboardSpecRenderer")
|
||||
? "playerLiveStoryboardSpecRenderer"
|
||||
: "playerStoryboardSpecRenderer"
|
||||
);
|
||||
|
||||
if (storyboardsRenderer == null) {
|
||||
return Collections.emptyList();
|
||||
|
|
@ -1283,15 +1304,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
|
||||
final String[] spec = storyboardsRendererSpec.split("\\|");
|
||||
final String url = spec[0];
|
||||
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
|
||||
final List<Frameset> result = new ArrayList<>(spec.length - 1);
|
||||
|
||||
for (int i = 1; i < spec.length; ++i) {
|
||||
final String[] parts = spec[i].split("#");
|
||||
if (parts.length != 8 || Integer.parseInt(parts[5]) == 0) {
|
||||
continue;
|
||||
}
|
||||
final int frameWidth = Integer.parseInt(parts[0]);
|
||||
final int frameHeight = Integer.parseInt(parts[1]);
|
||||
final int totalCount = Integer.parseInt(parts[2]);
|
||||
final int framesPerPageX = Integer.parseInt(parts[3]);
|
||||
final int framesPerPageY = Integer.parseInt(parts[4]);
|
||||
|
|
@ -1310,15 +1329,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
}
|
||||
result.add(new Frameset(
|
||||
urls,
|
||||
frameWidth,
|
||||
frameHeight,
|
||||
/*frameWidth=*/Integer.parseInt(parts[0]),
|
||||
/*frameHeight=*/Integer.parseInt(parts[1]),
|
||||
totalCount,
|
||||
Integer.parseInt(parts[5]),
|
||||
/*durationPerFrame=*/Integer.parseInt(parts[5]),
|
||||
framesPerPageX,
|
||||
framesPerPageY
|
||||
));
|
||||
}
|
||||
result.trimToSize();
|
||||
return result;
|
||||
} catch (final Exception e) {
|
||||
throw new ExtractionException("Could not get frames", e);
|
||||
|
|
@ -1328,8 +1346,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public Privacy getPrivacy() {
|
||||
final boolean isUnlisted = playerMicroFormatRenderer.getBoolean("isUnlisted");
|
||||
return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
|
||||
return playerMicroFormatRenderer.getBoolean("isUnlisted")
|
||||
? Privacy.UNLISTED
|
||||
: Privacy.PUBLIC;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
|
|
@ -1342,14 +1361,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public String getLicence() throws ParsingException {
|
||||
final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
|
||||
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
|
||||
.getObject("metadataRowContainer")
|
||||
.getObject("metadataRowContainerRenderer")
|
||||
.getArray("rows")
|
||||
.getObject(0).getObject("metadataRowRenderer");
|
||||
.getObject(0)
|
||||
.getObject("metadataRowRenderer");
|
||||
|
||||
final JsonArray contents = metadataRowRenderer.getArray("contents");
|
||||
final String license = getTextFromObject(contents.getObject(0));
|
||||
return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer
|
||||
.getObject("title"))) ? license : "YouTube licence";
|
||||
return license != null
|
||||
&& "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title")))
|
||||
? license
|
||||
: "YouTube licence";
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1367,32 +1390,43 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public List<StreamSegment> getStreamSegments() throws ParsingException {
|
||||
final ArrayList<StreamSegment> segments = new ArrayList<>();
|
||||
if (nextResponse.has("engagementPanels")) {
|
||||
final JsonArray panels = nextResponse.getArray("engagementPanels");
|
||||
JsonArray segmentsArray = null;
|
||||
|
||||
// Search for correct panel containing the data
|
||||
for (int i = 0; i < panels.size(); i++) {
|
||||
final String panelIdentifier = panels.getObject(i)
|
||||
if (!nextResponse.has("engagementPanels")) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
final JsonArray segmentsArray = nextResponse.getArray("engagementPanels")
|
||||
.stream()
|
||||
// Check if object is a JsonObject
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
// Check if the panel is the correct one
|
||||
.filter(panel -> "engagement-panel-macro-markers-description-chapters".equals(
|
||||
panel
|
||||
.getObject("engagementPanelSectionListRenderer")
|
||||
.getString("panelIdentifier");
|
||||
// panelIdentifier might be null if the panel has something to do with ads
|
||||
// See https://github.com/TeamNewPipe/NewPipe/issues/7792#issuecomment-1030900188
|
||||
if ("engagement-panel-macro-markers-description-chapters".equals(panelIdentifier)) {
|
||||
segmentsArray = panels.getObject(i)
|
||||
.getObject("engagementPanelSectionListRenderer").getObject("content")
|
||||
.getObject("macroMarkersListRenderer").getArray("contents");
|
||||
break;
|
||||
}
|
||||
.getString("panelIdentifier")))
|
||||
// Extract the data
|
||||
.map(panel -> panel
|
||||
.getObject("engagementPanelSectionListRenderer")
|
||||
.getObject("content")
|
||||
.getObject("macroMarkersListRenderer")
|
||||
.getArray("contents"))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
// If no data was found exit
|
||||
if (segmentsArray == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
if (segmentsArray != null) {
|
||||
final long duration = getLength();
|
||||
for (final Object object : segmentsArray) {
|
||||
final JsonObject segmentJson = ((JsonObject) object)
|
||||
.getObject("macroMarkersListItemRenderer");
|
||||
|
||||
final List<StreamSegment> segments = new ArrayList<>();
|
||||
for (final JsonObject segmentJson : segmentsArray.stream()
|
||||
.filter(JsonObject.class::isInstance)
|
||||
.map(JsonObject.class::cast)
|
||||
.map(object -> object.getObject("macroMarkersListItemRenderer"))
|
||||
.collect(Collectors.toList())
|
||||
) {
|
||||
final int startTimeSeconds = segmentJson.getObject("onTap")
|
||||
.getObject("watchEndpoint").getInt("startTimeSeconds", -1);
|
||||
|
||||
|
|
@ -1411,29 +1445,31 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
|
||||
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
|
||||
if (segmentJson.has("thumbnail")) {
|
||||
final JsonArray previewsArray = segmentJson.getObject("thumbnail")
|
||||
final JsonArray previewsArray = segmentJson
|
||||
.getObject("thumbnail")
|
||||
.getArray("thumbnails");
|
||||
if (!previewsArray.isEmpty()) {
|
||||
// Assume that the thumbnail with the highest resolution is at the
|
||||
// last position
|
||||
// Assume that the thumbnail with the highest resolution is at the last position
|
||||
final String url = previewsArray
|
||||
.getObject(previewsArray.size() - 1).getString("url");
|
||||
.getObject(previewsArray.size() - 1)
|
||||
.getString("url");
|
||||
segment.setPreviewUrl(fixThumbnailUrl(url));
|
||||
}
|
||||
}
|
||||
segments.add(segment);
|
||||
}
|
||||
}
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public List<MetaInfo> getMetaInfo() throws ParsingException {
|
||||
return YoutubeParsingHelper.getMetaInfo(
|
||||
nextResponse.getObject("contents").getObject("twoColumnWatchNextResults")
|
||||
.getObject("results").getObject("results").getArray("contents"));
|
||||
return YoutubeParsingHelper.getMetaInfo(nextResponse
|
||||
.getObject("contents")
|
||||
.getObject("twoColumnWatchNextResults")
|
||||
.getObject("results")
|
||||
.getObject("results")
|
||||
.getArray("contents"));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
package org.schabi.newpipe.extractor.stream;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
||||
/**
|
||||
* Creates a stream object from url, format and optional torrent url
|
||||
*/
|
||||
public abstract class Stream implements Serializable {
|
||||
private final MediaFormat mediaFormat;
|
||||
public final String url;
|
||||
public final String torrentUrl;
|
||||
private final String url;
|
||||
private final String torrentUrl;
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link #getFormat()} or {@link #getFormatId()}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.utils;
|
|||
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
|
@ -18,10 +16,17 @@ import java.util.Map;
|
|||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public final class Utils {
|
||||
|
||||
public static final String HTTP = "http://";
|
||||
public static final String HTTPS = "https://";
|
||||
/**
|
||||
* @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8}
|
||||
*/
|
||||
@Deprecated
|
||||
public static final String UTF_8 = "UTF-8";
|
||||
public static final String EMPTY_STRING = "";
|
||||
private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");
|
||||
|
|
|
|||
Loading…
Reference in a new issue