Apply code review and Streams rework

This commit is contained in:
Stypox 2022-05-28 00:26:53 +02:00 committed by TiA4f8R
parent d652e05874
commit b3c620f0d8
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
10 changed files with 248 additions and 351 deletions

View file

@ -159,11 +159,11 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
return getStreams("audio",
dto -> {
final AudioStream.Builder builder = new AudioStream.Builder()
.setId(dto.getUrlValue().getString("tech", ID_UNKNOWN))
.setContent(dto.getUrlValue().getString(URL), true)
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
.setContent(dto.urlValue.getString(URL), true)
.setAverageBitrate(UNKNOWN_BITRATE);
if ("hls".equals(dto.getUrlKey())) {
if ("hls".equals(dto.urlKey)) {
// We don't know with the type string what media format will
// have HLS streams.
// However, the tech string may contain some information
@ -172,7 +172,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
.build();
}
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.getUrlKey()))
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
.build();
});
}
@ -181,15 +181,15 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
return getStreams("video",
dto -> {
final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize");
final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
final VideoStream.Builder builder = new VideoStream.Builder()
.setId(dto.getUrlValue().getString("tech", ID_UNKNOWN))
.setContent(dto.getUrlValue().getString(URL), true)
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
.setContent(dto.urlValue.getString(URL), true)
.setIsVideoOnly(false)
.setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
if ("hls".equals(dto.getUrlKey())) {
if ("hls".equals(dto.urlKey)) {
// We don't know with the type string what media format will
// have HLS streams.
// However, the tech string may contain some information
@ -198,11 +198,32 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
.build();
}
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.getUrlKey()))
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
.build();
});
}
/**
* This is just an internal class used in {@link #getStreams(String, Function)} to tie together
* the stream json object, its URL key and its URL value. An object of this class would be
* temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
* object based on the wanted stream type.
*/
private static final class MediaCCCLiveStreamMapperDTO {
final JsonObject streamJsonObj;
final String urlKey;
final JsonObject urlValue;
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
final String urlKey,
final JsonObject urlValue) {
this.streamJsonObj = streamJsonObj;
this.urlKey = urlKey;
this.urlValue = urlValue;
}
}
private <T extends Stream> List<T> getStreams(
@Nonnull final String streamType,
@Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
@ -220,7 +241,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
e.getKey(),
(JsonObject) e.getValue())))
// The DASH manifest will be extracted with getDashMpdUrl
.filter(dto -> !"dash".equals(dto.getUrlKey()))
.filter(dto -> !"dash".equals(dto.urlKey))
// Convert
.map(converter)
.collect(Collectors.toList());

View file

@ -1,29 +0,0 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonObject;
final class MediaCCCLiveStreamMapperDTO {
private final JsonObject streamJsonObj;
private final String urlKey;
private final JsonObject urlValue;
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
final String urlKey,
final JsonObject urlValue) {
this.streamJsonObj = streamJsonObj;
this.urlKey = urlKey;
this.urlValue = urlValue;
}
JsonObject getStreamJsonObj() {
return streamJsonObj;
}
String getUrlKey() {
return urlKey;
}
JsonObject getUrlValue() {
return urlValue;
}
}

View file

@ -409,11 +409,7 @@ public class ItagItem implements Serializable {
* @param sampleRate the sample rate of an audio itag
*/
public void setSampleRate(final int sampleRate) {
if (sampleRate > 0) {
this.sampleRate = sampleRate;
} else {
this.sampleRate = SAMPLE_RATE_UNKNOWN;
}
this.sampleRate = sampleRate > 0 ? sampleRate : SAMPLE_RATE_UNKNOWN;
}
/**

View file

@ -97,6 +97,25 @@ public final class YoutubeDashManifestCreatorsUtils {
public static final String SEGMENT_BASE = "SegmentBase";
public static final String INITIALIZATION = "Initialization";
/**
* Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
* name and value, then add it to the provided element using {@link
* Element#setAttributeNode(Attr)}.
*
* @param element element to which to add the created node
* @param doc document to use to create the attribute
* @param name name of the attribute
* @param value value of the attribute, will be set using {@link Attr#setValue(String)}
*/
public static void setAttribute(final Element element,
final Document doc,
final String name,
final String value) {
final Attr attr = doc.createAttribute(name);
attr.setValue(value);
element.setAttributeNode(attr);
}
/**
* Generate a {@link Document} with common manifest creator elements added to it.
*
@ -123,17 +142,17 @@ public final class YoutubeDashManifestCreatorsUtils {
public static Document generateDocumentAndDoCommonElementsGeneration(
@Nonnull final ItagItem itagItem,
final long streamDuration) throws CreationException {
final Document document = generateDocumentAndMpdElement(streamDuration);
final Document doc = generateDocumentAndMpdElement(streamDuration);
generatePeriodElement(document);
generateAdaptationSetElement(document, itagItem);
generateRoleElement(document);
generateRepresentationElement(document, itagItem);
generatePeriodElement(doc);
generateAdaptationSetElement(doc, itagItem);
generateRoleElement(doc);
generateRepresentationElement(doc, itagItem);
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
generateAudioChannelConfigurationElement(document, itagItem);
generateAudioChannelConfigurationElement(doc, itagItem);
}
return document;
return doc;
}
/**
@ -161,46 +180,25 @@ public final class YoutubeDashManifestCreatorsUtils {
public static Document generateDocumentAndMpdElement(final long duration)
throws CreationException {
try {
final Document document = newDocument();
final Document doc = newDocument();
final Element mpdElement = document.createElement(MPD);
document.appendChild(mpdElement);
final Element mpdElement = doc.createElement(MPD);
doc.appendChild(mpdElement);
final Attr xmlnsXsiAttribute = document.createAttribute("xmlns:xsi");
xmlnsXsiAttribute.setValue("http://www.w3.org/2001/XMLSchema-instance");
mpdElement.setAttributeNode(xmlnsXsiAttribute);
setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
setAttribute(mpdElement, doc, "xsi:schemaLocation",
"urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
setAttribute(mpdElement, doc, "type", "static");
setAttribute(mpdElement, doc, "mediaPresentationDuration",
String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
final Attr xmlns = document.createAttribute("xmlns");
xmlns.setValue("urn:mpeg:DASH:schema:MPD:2011");
mpdElement.setAttributeNode(xmlns);
final Attr xsiSchemaLocationAttribute = document.createAttribute("xsi:schemaLocation");
xsiSchemaLocationAttribute.setValue("urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
mpdElement.setAttributeNode(xsiSchemaLocationAttribute);
final Attr minBufferTimeAttribute = document.createAttribute("minBufferTime");
minBufferTimeAttribute.setValue("PT1.500S");
mpdElement.setAttributeNode(minBufferTimeAttribute);
final Attr profilesAttribute = document.createAttribute("profiles");
profilesAttribute.setValue("urn:mpeg:dash:profile:full:2011");
mpdElement.setAttributeNode(profilesAttribute);
final Attr typeAttribute = document.createAttribute("type");
typeAttribute.setValue("static");
mpdElement.setAttributeNode(typeAttribute);
final Attr mediaPresentationDurationAttribute = document.createAttribute(
"mediaPresentationDuration");
final String durationSeconds = String.format(Locale.ENGLISH, "%.3f",
duration / 1000.0);
mediaPresentationDurationAttribute.setValue("PT" + durationSeconds + "S");
mpdElement.setAttributeNode(mediaPresentationDurationAttribute);
return document;
return doc;
} catch (final Exception e) {
throw new CreationException(
"Could not generate the DASH manifest or append the MPD document to it", e);
"Could not generate the DASH manifest or append the MPD doc to it", e);
}
}
@ -212,14 +210,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateDocumentAndMpdElement(long)}.
* </p>
*
* @param document the {@link Document} on which the the {@code <Period>} element will be
* appended
* @param doc the {@link Document} on which the the {@code <Period>} element will be appended
*/
public static void generatePeriodElement(@Nonnull final Document document)
public static void generatePeriodElement(@Nonnull final Document doc)
throws CreationException {
try {
final Element mpdElement = (Element) document.getElementsByTagName(MPD).item(0);
final Element periodElement = document.createElement(PERIOD);
final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
final Element periodElement = doc.createElement(PERIOD);
mpdElement.appendChild(periodElement);
} catch (final DOMException e) {
throw CreationException.couldNotAddElement(PERIOD, e);
@ -235,21 +232,18 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generatePeriodElement(Document)}.
* </p>
*
* @param document the {@link Document} on which the {@code <Period>} element will be
* appended
* @param doc the {@link Document} on which the {@code <Period>} element will be appended
* @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
*/
public static void generateAdaptationSetElement(@Nonnull final Document document,
public static void generateAdaptationSetElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem)
throws CreationException {
try {
final Element periodElement = (Element) document.getElementsByTagName(PERIOD)
final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
.item(0);
final Element adaptationSetElement = document.createElement(ADAPTATION_SET);
final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
final Attr idAttribute = document.createAttribute("id");
idAttribute.setValue("0");
adaptationSetElement.setAttributeNode(idAttribute);
setAttribute(adaptationSetElement, doc, "id", "0");
final MediaFormat mediaFormat = itagItem.getMediaFormat();
if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
@ -257,14 +251,8 @@ public final class YoutubeDashManifestCreatorsUtils {
"the MediaFormat or its mime type is null or empty");
}
final Attr mimeTypeAttribute = document.createAttribute("mimeType");
mimeTypeAttribute.setValue(mediaFormat.getMimeType());
adaptationSetElement.setAttributeNode(mimeTypeAttribute);
final Attr subsegmentAlignmentAttribute = document.createAttribute(
"subsegmentAlignment");
subsegmentAlignmentAttribute.setValue("true");
adaptationSetElement.setAttributeNode(subsegmentAlignmentAttribute);
setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
periodElement.appendChild(adaptationSetElement);
} catch (final DOMException e) {
@ -289,23 +277,17 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the the {@code <Role>} element will be
* appended
* @param doc the {@link Document} on which the the {@code <Role>} element will be appended
*/
public static void generateRoleElement(@Nonnull final Document document)
public static void generateRoleElement(@Nonnull final Document doc)
throws CreationException {
try {
final Element adaptationSetElement = (Element) document.getElementsByTagName(
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
ADAPTATION_SET).item(0);
final Element roleElement = document.createElement(ROLE);
final Element roleElement = doc.createElement(ROLE);
final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
schemeIdUriAttribute.setValue("urn:mpeg:DASH:role:2011");
roleElement.setAttributeNode(schemeIdUriAttribute);
final Attr valueAttribute = document.createAttribute("value");
valueAttribute.setValue("main");
roleElement.setAttributeNode(valueAttribute);
setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
setAttribute(roleElement, doc, "value", "main");
adaptationSetElement.appendChild(roleElement);
} catch (final DOMException e) {
@ -322,56 +304,43 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateAdaptationSetElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the the {@code <SegmentTimeline>} element will
* be appended
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
* appended
* @param itagItem the {@link ItagItem} to use, which must not be null
*/
public static void generateRepresentationElement(@Nonnull final Document document,
public static void generateRepresentationElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem)
throws CreationException {
try {
final Element adaptationSetElement = (Element) document.getElementsByTagName(
final Element adaptationSetElement = (Element) doc.getElementsByTagName(
ADAPTATION_SET).item(0);
final Element representationElement = document.createElement(REPRESENTATION);
final Element representationElement = doc.createElement(REPRESENTATION);
final int id = itagItem.id;
if (id <= 0) {
throw CreationException.couldNotAddElement(REPRESENTATION,
"the id of the ItagItem is <= 0");
}
final Attr idAttribute = document.createAttribute("id");
idAttribute.setValue(String.valueOf(id));
representationElement.setAttributeNode(idAttribute);
setAttribute(representationElement, doc, "id", String.valueOf(id));
final String codec = itagItem.getCodec();
if (isNullOrEmpty(codec)) {
throw CreationException.couldNotAddElement(ADAPTATION_SET,
"the codec value of the ItagItem is null or empty");
}
final Attr codecsAttribute = document.createAttribute("codecs");
codecsAttribute.setValue(codec);
representationElement.setAttributeNode(codecsAttribute);
final Attr startWithSAPAttribute = document.createAttribute("startWithSAP");
startWithSAPAttribute.setValue("1");
representationElement.setAttributeNode(startWithSAPAttribute);
final Attr maxPlayoutRateAttribute = document.createAttribute("maxPlayoutRate");
maxPlayoutRateAttribute.setValue("1");
representationElement.setAttributeNode(maxPlayoutRateAttribute);
setAttribute(representationElement, doc, "codecs", codec);
setAttribute(representationElement, doc, "startWithSAP", "1");
setAttribute(representationElement, doc, "maxPlayoutRate", "1");
final int bitrate = itagItem.getBitrate();
if (bitrate <= 0) {
throw CreationException.couldNotAddElement(REPRESENTATION,
"the bitrate of the ItagItem is <= 0");
}
final Attr bandwidthAttribute = document.createAttribute("bandwidth");
bandwidthAttribute.setValue(String.valueOf(bitrate));
representationElement.setAttributeNode(bandwidthAttribute);
setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
final ItagItem.ItagType itagType = itagItem.itagType;
if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
if (itagItem.itagType == ItagItem.ItagType.VIDEO
|| itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
final int height = itagItem.getHeight();
final int width = itagItem.getWidth();
if (height <= 0 && width <= 0) {
@ -380,25 +349,19 @@ public final class YoutubeDashManifestCreatorsUtils {
}
if (width > 0) {
final Attr widthAttribute = document.createAttribute("width");
widthAttribute.setValue(String.valueOf(width));
representationElement.setAttributeNode(widthAttribute);
setAttribute(representationElement, doc, "width", String.valueOf(width));
}
final Attr heightAttribute = document.createAttribute("height");
heightAttribute.setValue(String.valueOf(itagItem.getHeight()));
representationElement.setAttributeNode(heightAttribute);
setAttribute(representationElement, doc, "height",
String.valueOf(itagItem.getHeight()));
final int fps = itagItem.getFps();
if (fps > 0) {
final Attr frameRateAttribute = document.createAttribute("frameRate");
frameRateAttribute.setValue(String.valueOf(fps));
representationElement.setAttributeNode(frameRateAttribute);
setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
}
}
if (itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
final Attr audioSamplingRateAttribute = document.createAttribute(
if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
final Attr audioSamplingRateAttribute = doc.createAttribute(
"audioSamplingRate");
audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
}
@ -433,32 +396,28 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateRepresentationElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the {@code <AudioChannelConfiguration>}
* element will be appended
* @param doc the {@link Document} on which the {@code <AudioChannelConfiguration>} element will
* be appended
* @param itagItem the {@link ItagItem} to use, which must not be null
*/
public static void generateAudioChannelConfigurationElement(
@Nonnull final Document document,
@Nonnull final Document doc,
@Nonnull final ItagItem itagItem) throws CreationException {
try {
final Element representationElement = (Element) document.getElementsByTagName(
final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0);
final Element audioChannelConfigurationElement = document.createElement(
final Element audioChannelConfigurationElement = doc.createElement(
AUDIO_CHANNEL_CONFIGURATION);
final Attr schemeIdUriAttribute = document.createAttribute("schemeIdUri");
schemeIdUriAttribute.setValue(
setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
"urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
audioChannelConfigurationElement.setAttributeNode(schemeIdUriAttribute);
final Attr valueAttribute = document.createAttribute("value");
final int audioChannels = itagItem.getAudioChannels();
if (audioChannels <= 0) {
if (itagItem.getAudioChannels() <= 0) {
throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
+ audioChannels);
+ itagItem.getAudioChannels());
}
valueAttribute.setValue(String.valueOf(itagItem.getAudioChannels()));
audioChannelConfigurationElement.setAttributeNode(valueAttribute);
setAttribute(audioChannelConfigurationElement, doc, "value",
String.valueOf(itagItem.getAudioChannels()));
representationElement.appendChild(audioChannelConfigurationElement);
} catch (final DOMException e) {
@ -467,22 +426,22 @@ public final class YoutubeDashManifestCreatorsUtils {
}
/**
* Convert a DASH manifest {@link Document document} to a string and cache it.
* Convert a DASH manifest {@link Document doc} to a string and cache it.
*
* @param originalBaseStreamingUrl the original base URL of the stream
* @param document the document to be converted
* @param doc the doc to be converted
* @param manifestCreatorCache the {@link ManifestCreatorCache} on which store the string
* generated
* @return the DASH manifest {@link Document document} converted to a string
* @return the DASH manifest {@link Document doc} converted to a string
*/
public static String buildAndCacheResult(
@Nonnull final String originalBaseStreamingUrl,
@Nonnull final Document document,
@Nonnull final Document doc,
@Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache)
throws CreationException {
try {
final String documentXml = documentToXml(document);
final String documentXml = documentToXml(doc);
manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
return documentXml;
} catch (final Exception e) {
@ -517,13 +476,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateRepresentationElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the {@code <SegmentTemplate>} element will
* @param doc the {@link Document} on which the {@code <SegmentTemplate>} element will
* be appended
* @param baseUrl the base URL of the OTF/post-live-DVR stream
* @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
* {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
*/
public static void generateSegmentTemplateElement(@Nonnull final Document document,
public static void generateSegmentTemplateElement(@Nonnull final Document doc,
@Nonnull final String baseUrl,
final DeliveryType deliveryType)
throws CreationException {
@ -533,32 +492,22 @@ public final class YoutubeDashManifestCreatorsUtils {
}
try {
final Element representationElement = (Element) document.getElementsByTagName(
final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0);
final Element segmentTemplateElement = document.createElement(SEGMENT_TEMPLATE);
final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
final Attr startNumberAttribute = document.createAttribute("startNumber");
final boolean isDeliveryTypeLive = deliveryType == DeliveryType.LIVE;
// The first sequence of post DVR streams is the beginning of the video stream and not
// an initialization segment
final String startNumberValue = isDeliveryTypeLive ? "0" : "1";
startNumberAttribute.setValue(startNumberValue);
segmentTemplateElement.setAttributeNode(startNumberAttribute);
final Attr timescaleAttribute = document.createAttribute("timescale");
timescaleAttribute.setValue("1000");
segmentTemplateElement.setAttributeNode(timescaleAttribute);
setAttribute(segmentTemplateElement, doc, "startNumber",
deliveryType == DeliveryType.LIVE ? "0" : "1");
setAttribute(segmentTemplateElement, doc, "timescale", "1000");
// Post-live-DVR/ended livestreams streams don't require an initialization sequence
if (!isDeliveryTypeLive) {
final Attr initializationAttribute = document.createAttribute("initialization");
initializationAttribute.setValue(baseUrl + SQ_0);
segmentTemplateElement.setAttributeNode(initializationAttribute);
if (deliveryType != DeliveryType.LIVE) {
setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
}
final Attr mediaAttribute = document.createAttribute("media");
mediaAttribute.setValue(baseUrl + "&sq=$Number$");
segmentTemplateElement.setAttributeNode(mediaAttribute);
setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
representationElement.appendChild(segmentTemplateElement);
} catch (final DOMException e) {
@ -575,15 +524,15 @@ public final class YoutubeDashManifestCreatorsUtils {
* {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
* </p>
*
* @param document the {@link Document} on which the the {@code <SegmentTimeline>} element will
* be appended
* @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
* appended
*/
public static void generateSegmentTimelineElement(@Nonnull final Document document)
public static void generateSegmentTimelineElement(@Nonnull final Document doc)
throws CreationException {
try {
final Element segmentTemplateElement = (Element) document.getElementsByTagName(
final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
SEGMENT_TEMPLATE).item(0);
final Element segmentTimelineElement = document.createElement(SEGMENT_TIMELINE);
final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
segmentTemplateElement.appendChild(segmentTimelineElement);
} catch (final DOMException e) {
@ -672,8 +621,7 @@ public final class YoutubeDashManifestCreatorsUtils {
// supported by all platforms (like the Android implementation)
}
final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
return documentBuilder.newDocument();
return documentBuilderFactory.newDocumentBuilder().newDocument();
}
/**
@ -681,13 +629,13 @@ public final class YoutubeDashManifestCreatorsUtils {
* support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
* {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
*
* @param document the document to convert, which must have been created using
* {@link #newDocument()} to properly prevent XXE attacks
* @return the document converted to an XML string, making sure there can't be XXE attacks
* @param doc the doc to convert, which must have been created using {@link #newDocument()} to
* properly prevent XXE attacks
* @return the doc converted to an XML string, making sure there can't be XXE attacks
*/
// Sonar warning is suppressed because it is still shown even if we apply its solution
@SuppressWarnings("squid:S2755")
private static String documentToXml(@Nonnull final Document document)
private static String documentToXml(@Nonnull final Document doc)
throws TransformerException {
final TransformerFactory transformerFactory = TransformerFactory.newInstance();
@ -705,7 +653,7 @@ public final class YoutubeDashManifestCreatorsUtils {
transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
final StringWriter result = new StringWriter();
transformer.transform(new DOMSource(document), new StreamResult(result));
transformer.transform(new DOMSource(doc), new StreamResult(result));
return result.toString();
}

View file

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.schabi.newpipe.extractor.utils.Utils;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -23,6 +22,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
@ -148,14 +148,14 @@ public final class YoutubeOtfDashManifestCreator {
streamDuration = durationSecondsFallback * 1000;
}
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration);
generateSegmentTemplateElement(document, realOtfBaseStreamingUrl, DeliveryType.OTF);
generateSegmentTimelineElement(document);
generateSegmentElementsForOtfStreams(segmentDuration, document);
generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
generateSegmentTimelineElement(doc);
generateSegmentElementsForOtfStreams(segmentDuration, doc);
return buildAndCacheResult(otfBaseStreamingUrl, document, OTF_STREAMS_CACHE);
return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
}
/**
@ -192,17 +192,18 @@ public final class YoutubeOtfDashManifestCreator {
*
* @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
* regular expressions
* @param document the {@link Document} on which the {@code <S>} elements will be appended
* @param doc the {@link Document} on which the {@code <S>} elements will be
* appended
*/
private static void generateSegmentElementsForOtfStreams(
@Nonnull final String[] segmentDurations,
@Nonnull final Document document) throws CreationException {
@Nonnull final Document doc) throws CreationException {
try {
final Element segmentTimelineElement = (Element) document.getElementsByTagName(
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
SEGMENT_TIMELINE).item(0);
for (final String segmentDuration : segmentDurations) {
final Element sElement = document.createElement("S");
final Element sElement = doc.createElement("S");
final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
// make sure segmentLengthRepeat[0], which is the length, is convertible to int
@ -212,14 +213,9 @@ public final class YoutubeOtfDashManifestCreator {
if (segmentLengthRepeat.length > 1) {
final int segmentRepeatCount = Integer.parseInt(
Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
final Attr rAttribute = document.createAttribute("r");
rAttribute.setValue(String.valueOf(segmentRepeatCount));
sElement.setAttributeNode(rAttribute);
setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
}
final Attr dAttribute = document.createAttribute("d");
dAttribute.setValue(segmentLengthRepeat[0]);
sElement.setAttributeNode(dAttribute);
setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
segmentTimelineElement.appendChild(sElement);
}

View file

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -23,6 +22,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -159,15 +159,15 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
streamDuration = durationSecondsFallback;
}
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration);
generateSegmentTemplateElement(document, realPostLiveStreamDvrStreamingUrl,
generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
DeliveryType.LIVE);
generateSegmentTimelineElement(document);
generateSegmentElementForPostLiveDvrStreams(document, targetDurationSec, segmentCount);
generateSegmentTimelineElement(doc);
generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
return buildAndCacheResult(postLiveStreamDvrStreamingUrl, document,
return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
POST_LIVE_DVR_STREAMS_CACHE);
}
@ -190,7 +190,7 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
* {@code <S d="targetDurationSecValue" r="segmentCount" />}
* </p>
*
* @param document the {@link Document} on which the {@code <S>} element will
* @param doc the {@link Document} on which the {@code <S>} element will
* be appended
* @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
* response's stream
@ -198,21 +198,16 @@ public final class YoutubePostLiveStreamDvrDashManifestCreator {
* #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
*/
private static void generateSegmentElementForPostLiveDvrStreams(
@Nonnull final Document document,
@Nonnull final Document doc,
final int targetDurationSeconds,
@Nonnull final String segmentCount) throws CreationException {
try {
final Element segmentTimelineElement = (Element) document.getElementsByTagName(
final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
SEGMENT_TIMELINE).item(0);
final Element sElement = document.createElement("S");
final Element sElement = doc.createElement("S");
final Attr dAttribute = document.createAttribute("d");
dAttribute.setValue(String.valueOf(targetDurationSeconds * 1000));
sElement.setAttributeNode(dAttribute);
final Attr rAttribute = document.createAttribute("r");
rAttribute.setValue(segmentCount);
sElement.setAttributeNode(rAttribute);
setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
setAttribute(sElement, doc, "r", segmentCount);
segmentTimelineElement.appendChild(sElement);
} catch (final DOMException e) {

View file

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
import org.w3c.dom.Attr;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -18,6 +17,7 @@ import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
/**
* Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
@ -100,14 +100,14 @@ public final class YoutubeProgressiveDashManifestCreator {
}
}
final Document document = generateDocumentAndDoCommonElementsGeneration(itagItem,
final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
streamDuration);
generateBaseUrlElement(document, progressiveStreamingBaseUrl);
generateSegmentBaseElement(document, itagItem);
generateInitializationElement(document, itagItem);
generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
generateSegmentBaseElement(doc, itagItem);
generateInitializationElement(doc, itagItem);
return buildAndCacheResult(progressiveStreamingBaseUrl, document,
return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
PROGRESSIVE_STREAMS_CACHE);
}
@ -128,18 +128,17 @@ public final class YoutubeProgressiveDashManifestCreator {
* {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the {@code <BaseURL>} element will
* be appended
* @param doc the {@link Document} on which the {@code <BaseURL>} element will be appended
* @param baseUrl the base URL of the stream, which must not be null and will be set as the
* content of the {@code <BaseURL>} element
*/
private static void generateBaseUrlElement(@Nonnull final Document document,
private static void generateBaseUrlElement(@Nonnull final Document doc,
@Nonnull final String baseUrl)
throws CreationException {
try {
final Element representationElement = (Element) document.getElementsByTagName(
final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0);
final Element baseURLElement = document.createElement(BASE_URL);
final Element baseURLElement = doc.createElement(BASE_URL);
baseURLElement.setTextContent(baseUrl);
representationElement.appendChild(baseURLElement);
} catch (final DOMException e) {
@ -167,28 +166,23 @@ public final class YoutubeProgressiveDashManifestCreator {
* should be generated too.
* </p>
*
* @param document the {@link Document} on which the {@code <SegmentBase>} element will be
* appended
* @param doc the {@link Document} on which the {@code <SegmentBase>} element will be appended
* @param itagItem the {@link ItagItem} to use, which must not be null
*/
private static void generateSegmentBaseElement(@Nonnull final Document document,
private static void generateSegmentBaseElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem)
throws CreationException {
try {
final Element representationElement = (Element) document.getElementsByTagName(
final Element representationElement = (Element) doc.getElementsByTagName(
REPRESENTATION).item(0);
final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
final Element segmentBaseElement = document.createElement(SEGMENT_BASE);
final Attr indexRangeAttribute = document.createAttribute("indexRange");
final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
throw CreationException.couldNotAddElement(SEGMENT_BASE,
"ItagItem's indexStart or " + "indexEnd are < 0: "
+ itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
"ItagItem's indexStart or " + "indexEnd are < 0: " + range);
}
indexRangeAttribute.setValue(itagItem.getIndexStart() + "-" + itagItem.getIndexEnd());
segmentBaseElement.setAttributeNode(indexRangeAttribute);
setAttribute(segmentBaseElement, doc, "indexRange", range);
representationElement.appendChild(segmentBaseElement);
} catch (final DOMException e) {
@ -214,28 +208,24 @@ public final class YoutubeProgressiveDashManifestCreator {
* {@link #generateSegmentBaseElement(Document, ItagItem)}).
* </p>
*
* @param document the {@link Document} on which the {@code <Initialization>} element will
* be appended
* @param doc the {@link Document} on which the {@code <Initialization>} element will be
* appended
* @param itagItem the {@link ItagItem} to use, which must not be null
*/
private static void generateInitializationElement(@Nonnull final Document document,
private static void generateInitializationElement(@Nonnull final Document doc,
@Nonnull final ItagItem itagItem)
throws CreationException {
try {
final Element segmentBaseElement = (Element) document.getElementsByTagName(
final Element segmentBaseElement = (Element) doc.getElementsByTagName(
SEGMENT_BASE).item(0);
final Element initializationElement = doc.createElement(INITIALIZATION);
final Element initializationElement = document.createElement(INITIALIZATION);
final Attr rangeAttribute = document.createAttribute("range");
final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
throw CreationException.couldNotAddElement(INITIALIZATION,
"ItagItem's initStart and/or " + "initEnd are/is < 0: "
+ itagItem.getInitStart() + "-" + itagItem.getInitEnd());
"ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
}
rangeAttribute.setValue(itagItem.getInitStart() + "-" + itagItem.getInitEnd());
initializationElement.setAttributeNode(rangeAttribute);
setAttribute(initializationElement, doc, "range", range);
segmentBaseElement.appendChild(initializationElement);
} catch (final DOMException e) {

View file

@ -1147,34 +1147,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final java.util.function.Function<ItagInfo, T> streamBuilderHelper,
final String streamTypeExceptionMessage) throws ParsingException {
try {
final List<ItagInfo> itagInfos = new ArrayList<>();
if (html5StreamingData == null && androidStreamingData == null
&& iosStreamingData == null) {
return Collections.emptyList();
}
final List<Pair<JsonObject, String>> streamingDataAndCpnLoopList = new ArrayList<>();
// Use the androidStreamingData object first because there is no n param and no
// signatureCiphers in streaming URLs of the Android client
streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn));
streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn));
// Use the iosStreamingData object in the last position because most of the available
// streams can be extracted with the Android and web clients and also because the iOS
// client is only enabled by default on livestreams
streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn));
for (final Pair<JsonObject, String> pair : streamingDataAndCpnLoopList) {
itagInfos.addAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey,
itagTypeWanted, pair.getSecond()));
}
final String videoId = getId();
final List<T> streamList = new ArrayList<>();
for (final ItagInfo itagInfo : itagInfos) {
final T stream = streamBuilderHelper.apply(itagInfo);
if (!Stream.containSimilarStream(stream, streamList)) {
streamList.add(stream);
}
}
java.util.stream.Stream.of(
// Use the androidStreamingData object first because there is no n param and no
// signatureCiphers in streaming URLs of the Android client
new Pair<>(androidStreamingData, androidCpn),
new Pair<>(html5StreamingData, html5Cpn),
// Use the iosStreamingData object in the last position because most of the
// available streams can be extracted with the Android and web clients and also
// because the iOS client is only enabled by default on livestreams
new Pair<>(iosStreamingData, iosCpn)
)
.flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
streamingDataKey, itagTypeWanted, pair.getSecond()))
.map(streamBuilderHelper)
.forEachOrdered(stream -> {
if (!Stream.containSimilarStream(stream, streamList)) {
streamList.add(stream);
}
});
return streamList;
} catch (final Exception e) {
@ -1293,43 +1286,36 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private List<ItagInfo> getStreamsFromStreamingDataKey(
private java.util.stream.Stream<ItagInfo> getStreamsFromStreamingDataKey(
final String videoId,
final JsonObject streamingData,
final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted,
@Nonnull final String contentPlaybackNonce) throws ParsingException {
@Nonnull final String contentPlaybackNonce) {
if (streamingData == null || !streamingData.has(streamingDataKey)) {
return Collections.emptyList();
return java.util.stream.Stream.empty();
}
final String videoId = getId();
final List<ItagInfo> itagInfos = new ArrayList<>();
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
final JsonObject formatData = formats.getObject(i);
final int itag = formatData.getInt("itag");
if (!ItagItem.isSupported(itag)) {
continue;
}
try {
final ItagItem itagItem = ItagItem.getItag(itag);
final ItagItem.ItagType itagType = itagItem.itagType;
if (itagType == itagTypeWanted) {
buildAndAddItagInfoToList(videoId, itagInfos, formatData, itagItem,
itagType, contentPlaybackNonce);
}
} catch (final IOException | ExtractionException ignored) {
}
}
return itagInfos;
return streamingData.getArray(streamingDataKey).stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(formatData -> {
try {
final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
if (itagItem.itagType == itagTypeWanted) {
return buildAndAddItagInfoToList(videoId, formatData, itagItem,
itagItem.itagType, contentPlaybackNonce);
}
} catch (final IOException | ExtractionException ignored) {
// if the itag is not supported and getItag fails, we end up here
}
return null;
})
.filter(Objects::nonNull);
}
private void buildAndAddItagInfoToList(
private ItagInfo buildAndAddItagInfoToList(
@Nonnull final String videoId,
@Nonnull final List<ItagInfo> itagInfos,
@Nonnull final JsonObject formatData,
@Nonnull final ItagItem itagItem,
@Nonnull final ItagItem.ItagType itagType,
@ -1372,12 +1358,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) {
itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec"));
}
if (itagType == ItagItem.ItagType.VIDEO || itagType == ItagItem.ItagType.VIDEO_ONLY) {
} else if (itagType == ItagItem.ItagType.VIDEO
|| itagType == ItagItem.ItagType.VIDEO_ONLY) {
itagItem.setFps(formatData.getInt("fps"));
}
if (itagType == ItagItem.ItagType.AUDIO) {
} else if (itagType == ItagItem.ItagType.AUDIO) {
// YouTube return the audio sample rate as a string
itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
itagItem.setAudioChannels(formatData.getInt("audioChannels"));
@ -1403,7 +1387,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM);
}
itagInfos.add(itagInfo);
return itagInfo;
}
@Nonnull

View file

@ -94,24 +94,15 @@ public abstract class Stream implements Serializable {
* Note: This method always returns false if the stream passed is null.
* </p>
*
* @param cmp the stream object to be compared to this stream object
* @param other the stream object to be compared to this stream object
* @return whether the stream have the same stats or not, based on the criteria above
*/
public boolean equalStats(@Nullable final Stream cmp) {
if (cmp == null) {
public boolean equalStats(@Nullable final Stream other) {
if (other == null || mediaFormat == null || other.mediaFormat == null) {
return false;
}
Boolean haveSameMediaFormatId = null;
if (mediaFormat != null && cmp.mediaFormat != null) {
haveSameMediaFormatId = mediaFormat.id == cmp.mediaFormat.id;
}
final boolean areUsingSameDeliveryMethodAndAreUrlStreams =
deliveryMethod == cmp.deliveryMethod && isUrl == cmp.isUrl;
return haveSameMediaFormatId != null
? haveSameMediaFormatId && areUsingSameDeliveryMethodAndAreUrlStreams
: areUsingSameDeliveryMethodAndAreUrlStreams;
return mediaFormat.id == other.mediaFormat.id && deliveryMethod == other.deliveryMethod
&& isUrl == other.isUrl;
}
/**

View file

@ -53,6 +53,11 @@ class ManifestCreatorCacheTest {
+ "call");
}
/**
* Adds sample strings to the provided manifest creator cache, in order to test clear factor and
* maximum size.
* @param cache the cache to fill with some data
*/
private static void setCacheContent(final ManifestCreatorCache<String, String> cache) {
int i = 0;
while (i < 26) {