From 881969f1da6da0e123e3aa51c80351bad34ceb6a Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Thu, 3 Mar 2022 20:46:53 +0100 Subject: [PATCH] Apply changes in all StreamExtractors except YouTube's one and fix extraction of PeerTube audio streams as video streams Some code in these classes has been also refactored/improved/optimized. Also fix the extraction of PeerTube audio streams as video streams, which are now returned as audio streams. --- .../BandcampRadioStreamExtractor.java | 44 +- .../extractors/BandcampStreamExtractor.java | 63 ++- .../MediaCCCLiveStreamExtractor.java | 175 ++++++-- .../extractors/MediaCCCStreamExtractor.java | 37 +- .../extractors/PeertubeStreamExtractor.java | 409 ++++++++++++++---- .../extractors/SoundcloudStreamExtractor.java | 136 ++++-- 6 files changed, 629 insertions(+), 235 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java index 639d2abb..7389d423 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java @@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; @@ -30,9 +29,12 @@ import java.util.List; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { + private static final String OPUS_LO = "opus_lo"; + private static final String MP3_128 = "mp3-128"; private JsonObject showInfo; public BandcampRadioStreamExtractor(final StreamingService service, @@ -78,11 +80,9 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Nonnull @Override - public String getUploaderName() throws ParsingException { - return Jsoup.parse(showInfo.getString("image_caption")).getElementsByTag("a").stream() - .map(Element::text) - .findFirst() - .orElseThrow(() -> new ParsingException("Could not get uploader name")); + public String getUploaderName() { + return Jsoup.parse(showInfo.getString("image_caption")) + .getElementsByTag("a").first().text(); } @Nullable @@ -116,23 +116,25 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Override public List getAudioStreams() { - final ArrayList list = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); final JsonObject streams = showInfo.getObject("audio_stream"); - if (streams.has("opus-lo")) { - list.add(new AudioStream( - streams.getString("opus-lo"), - MediaFormat.OPUS, 100 - )); - } - if (streams.has("mp3-128")) { - list.add(new AudioStream( - streams.getString("mp3-128"), - MediaFormat.MP3, 128 - )); + if (streams.has(MP3_128)) { + audioStreams.add(new AudioStream.Builder() + .setId(MP3_128) + .setContent(streams.getString(MP3_128), true) + .setMediaFormat(MediaFormat.MP3) + .setAverageBitrate(128) + .build()); + } else if (streams.has(OPUS_LO)) { + audioStreams.add(new AudioStream.Builder() + .setId(OPUS_LO) + .setContent(streams.getString(OPUS_LO), true) + .setMediaFormat(MediaFormat.OPUS) + .setAverageBitrate(100).build()); } - return list; + return audioStreams; } @Nonnull @@ -156,14 +158,14 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor { @Override public String getLicence() { // Contrary to other Bandcamp streams, radio streams don't have a license - return ""; + return EMPTY_STRING; } @Nonnull @Override public String getCategory() { // Contrary to other Bandcamp streams, radio streams don't have categories - return ""; + return EMPTY_STRING; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java index 896644e9..9bb6d5c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParserException; @@ -10,7 +12,6 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -27,16 +28,15 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import java.util.stream.Collectors; public class BandcampStreamExtractor extends StreamExtractor { - private JsonObject albumJson; private JsonObject current; private Document document; @@ -88,7 +88,7 @@ public class BandcampStreamExtractor extends StreamExtractor { public String getUploaderUrl() throws ParsingException { final String[] parts = getUrl().split("/"); // https: (/) (/) * .bandcamp.com (/) and leave out the rest - return "https://" + parts[2] + "/"; + return HTTPS + parts[2] + "/"; } @Nonnull @@ -119,7 +119,7 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public String getThumbnailUrl() throws ParsingException { if (albumJson.isNull("art_id")) { - return Utils.EMPTY_STRING; + return EMPTY_STRING; } else { return getImageUrl(albumJson.getLong("art_id"), true); } @@ -139,24 +139,26 @@ public class BandcampStreamExtractor extends StreamExtractor { public Description getDescription() { final String s = Utils.nonEmptyAndNullJoin( "\n\n", - new String[]{ + new String[] { current.getString("about"), current.getString("lyrics"), current.getString("credits") - } - ); + }); return new Description(s, Description.PLAIN_TEXT); } @Override public List getAudioStreams() { final List audioStreams = new ArrayList<>(); - - audioStreams.add(new AudioStream( - albumJson.getArray("trackinfo").getObject(0) - .getObject("file").getString("mp3-128"), - MediaFormat.MP3, 128 - )); + audioStreams.add(new AudioStream.Builder() + .setId("mp3-128") + .setContent(albumJson.getArray("trackinfo") + .getObject(0) + .getObject("file") + .getString("mp3-128"), true) + .setMediaFormat(MediaFormat.MP3) + .setAverageBitrate(128) + .build()); return audioStreams; } @@ -184,11 +186,11 @@ public class BandcampStreamExtractor extends StreamExtractor { @Override public PlaylistInfoItemsCollector getRelatedItems() { final PlaylistInfoItemsCollector collector = new PlaylistInfoItemsCollector(getServiceId()); - final Elements recommendedAlbums = document.getElementsByClass("recommended-album"); + document.getElementsByClass("recommended-album") + .stream() + .map(BandcampRelatedPlaylistInfoItemExtractor::new) + .forEach(collector::commit); - for (final Element album : recommendedAlbums) { - collector.commit(new BandcampRelatedPlaylistInfoItemExtractor(album)); - } return collector; } @@ -200,15 +202,17 @@ public class BandcampStreamExtractor extends StreamExtractor { .flatMap(element -> element.getElementsByClass("tag").stream()) .map(Element::text) .findFirst() - .orElse(""); + .orElse(EMPTY_STRING); } @Nonnull @Override public String getLicence() { - /* Tests resulted in this mapping of ints to licence: + /* + Tests resulted in this mapping of ints to licence: https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview (screenshot from a Bandcamp artist's - account) */ + account) + */ switch (current.getInt("license_type")) { case 1: @@ -233,14 +237,9 @@ public class BandcampStreamExtractor extends StreamExtractor { @Nonnull @Override public List getTags() { - final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords"); - - final List tags = new ArrayList<>(); - - for (final Element e : tagElements) { - tags.add(e.text()); - } - - return tags; + return document.getElementsByAttributeValue("itemprop", "keywords") + .stream() + .map(Element::text) + .collect(Collectors.toList()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java index 2a4eb45e..54c2d056 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; @@ -17,11 +18,21 @@ import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.stream.IntStream; import javax.annotation.Nonnull; +import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; +import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; + public class MediaCCCLiveStreamExtractor extends StreamExtractor { + private static final String STREAMS = "streams"; + private static final String URLS = "urls"; + private static final String URL = "url"; + private JsonObject conference = null; private String group = ""; private JsonObject room = null; @@ -34,19 +45,22 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - final JsonArray doc = - MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization()); - // find correct room + final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader, + getExtractorLocalization()); + // Find the correct room for (int c = 0; c < doc.size(); c++) { - conference = doc.getObject(c); - final JsonArray groups = conference.getArray("groups"); + final JsonObject conferenceObject = doc.getObject(c); + final JsonArray groups = conferenceObject.getArray("groups"); for (int g = 0; g < groups.size(); g++) { - group = groups.getObject(g).getString("group"); + final String groupObject = groups.getObject(g).getString("group"); final JsonArray rooms = groups.getObject(g).getArray("rooms"); for (int r = 0; r < rooms.size(); r++) { - room = rooms.getObject(r); - if (getId().equals( - conference.getString("slug") + "/" + room.getString("slug"))) { + final JsonObject roomObject = rooms.getObject(r); + if (getId().equals(conferenceObject.getString("slug") + "/" + + roomObject.getString("slug"))) { + this.conference = conferenceObject; + this.group = groupObject; + this.room = roomObject; return; } } @@ -91,69 +105,136 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor { return conference.getString("conference"); } + /** + * Get the URL of the first DASH stream found. + * + *

+ * There can be several DASH streams, so the URL of the first found is returned by this method. + *

+ * + *

+ * You can find the other video DASH streams by using {@link #getVideoStreams()} + *

+ */ + @Nonnull + @Override + public String getDashMpdUrl() throws ParsingException { + + for (int s = 0; s < room.getArray(STREAMS).size(); s++) { + final JsonObject stream = room.getArray(STREAMS).getObject(s); + final JsonObject urls = stream.getObject(URLS); + if (urls.has("dash")) { + return urls.getObject("dash").getString(URL, EMPTY_STRING); + } + } + + return EMPTY_STRING; + } + + /** + * Get the URL of the first HLS stream found. + * + *

+ * There can be several HLS streams, so the URL of the first found is returned by this method. + *

+ * + *

+ * You can find the other video HLS streams by using {@link #getVideoStreams()} + *

+ */ @Nonnull @Override public String getHlsUrl() { - // TODO: There are multiple HLS streams. - // Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams, - // so the user can choose a resolution. - for (int s = 0; s < room.getArray("streams").size(); s++) { - final JsonObject stream = room.getArray("streams").getObject(s); - if (stream.getString("type").equals("video")) { - if (stream.has("hls")) { - return stream.getObject("urls").getObject("hls").getString("url"); - } + for (int s = 0; s < room.getArray(STREAMS).size(); s++) { + final JsonObject stream = room.getArray(STREAMS).getObject(s); + final JsonObject urls = stream.getObject(URLS); + if (urls.has("hls")) { + return urls.getObject("hls").getString(URL, EMPTY_STRING); } } - return ""; + return EMPTY_STRING; } @Override public List getAudioStreams() throws IOException, ExtractionException { final List audioStreams = new ArrayList<>(); - for (int s = 0; s < room.getArray("streams").size(); s++) { - final JsonObject stream = room.getArray("streams").getObject(s); - if (stream.getString("type").equals("audio")) { - for (final String type : stream.getObject("urls").keySet()) { - final JsonObject url = stream.getObject("urls").getObject(type); - audioStreams.add(new AudioStream(url.getString("url"), - MediaFormat.getFromSuffix(type), -1)); - } - } - } + IntStream.range(0, room.getArray(STREAMS).size()) + .mapToObj(s -> room.getArray(STREAMS).getObject(s)) + .filter(streamJsonObject -> streamJsonObject.getString("type").equals("audio")) + .forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet() + .forEach(type -> { + final JsonObject urlObject = streamJsonObject.getObject(URLS) + .getObject(type); + // The DASH manifest will be extracted with getDashMpdUrl + if (!type.equals("dash")) { + final AudioStream.Builder builder = new AudioStream.Builder() + .setId(urlObject.getString("tech", ID_UNKNOWN)) + .setContent(urlObject.getString(URL), true) + .setAverageBitrate(UNKNOWN_BITRATE); + if (type.equals("hls")) { + // We don't know with the type string what media format will + // have HLS streams. + // However, the tech string may contain some information + // about the media format used. + builder.setDeliveryMethod(DeliveryMethod.HLS); + } else { + builder.setMediaFormat(MediaFormat.getFromSuffix(type)); + } + + audioStreams.add(builder.build()); + } + })); + return audioStreams; } @Override public List getVideoStreams() throws IOException, ExtractionException { final List videoStreams = new ArrayList<>(); - for (int s = 0; s < room.getArray("streams").size(); s++) { - final JsonObject stream = room.getArray("streams").getObject(s); - if (stream.getString("type").equals("video")) { - final String resolution = stream.getArray("videoSize").getInt(0) + "x" - + stream.getArray("videoSize").getInt(1); - for (final String type : stream.getObject("urls").keySet()) { - if (!type.equals("hls")) { - final JsonObject url = stream.getObject("urls").getObject(type); - videoStreams.add(new VideoStream( - url.getString("url"), - MediaFormat.getFromSuffix(type), - resolution)); - } - } - } - } + IntStream.range(0, room.getArray(STREAMS).size()) + .mapToObj(s -> room.getArray(STREAMS).getObject(s)) + .filter(stream -> stream.getString("type").equals("video")) + .forEachOrdered(streamJsonObject -> streamJsonObject.getObject(URLS).keySet() + .forEach(type -> { + final String resolution = + streamJsonObject.getArray("videoSize").getInt(0) + + "x" + + streamJsonObject.getArray("videoSize").getInt(1); + final JsonObject urlObject = streamJsonObject.getObject(URLS) + .getObject(type); + // The DASH manifest will be extracted with getDashMpdUrl + if (!type.equals("dash")) { + final VideoStream.Builder builder = new VideoStream.Builder() + .setId(urlObject.getString("tech", ID_UNKNOWN)) + .setContent(urlObject.getString(URL), true) + .setIsVideoOnly(false) + .setResolution(resolution); + + if (type.equals("hls")) { + // We don't know with the type string what media format will + // have HLS streams. + // However, the tech string may contain some information + // about the media format used. + builder.setDeliveryMethod(DeliveryMethod.HLS); + } else { + builder.setMediaFormat(MediaFormat.getFromSuffix(type)); + } + + videoStreams.add(builder.build()); + } + })); + return videoStreams; } @Override public List getVideoOnlyStreams() { - return null; + return Collections.emptyList(); } @Override public StreamType getStreamType() throws ParsingException { - return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available + return StreamType.LIVE_STREAM; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 64a26897..53cc53ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -1,5 +1,8 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors; +import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; +import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -99,7 +102,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); if (mimeType.startsWith("audio")) { - //first we need to resolve the actual video data from CDN + // First we need to resolve the actual video data from CDN final MediaFormat mediaFormat; if (mimeType.endsWith("opus")) { mediaFormat = MediaFormat.OPUS; @@ -108,11 +111,18 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } else if (mimeType.endsWith("ogg")) { mediaFormat = MediaFormat.OGG; } else { - throw new ExtractionException("Unknown media format: " + mimeType); + mediaFormat = null; } - audioStreams.add(new AudioStream(recording.getString("recording_url"), - mediaFormat, -1)); + // Don't use the containsSimilarStream method because it will always return + // false so if there are multiples audio streams available, only the first will + // be extracted in this case. + audioStreams.add(new AudioStream.Builder() + .setId(recording.getString("filename", ID_UNKNOWN)) + .setContent(recording.getString("recording_url"), true) + .setMediaFormat(mediaFormat) + .setAverageBitrate(UNKNOWN_BITRATE) + .build()); } } return audioStreams; @@ -126,7 +136,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { final JsonObject recording = recordings.getObject(i); final String mimeType = recording.getString("mime_type"); if (mimeType.startsWith("video")) { - //first we need to resolve the actual video data from CDN + // First we need to resolve the actual video data from CDN final MediaFormat mediaFormat; if (mimeType.endsWith("webm")) { @@ -134,13 +144,21 @@ public class MediaCCCStreamExtractor extends StreamExtractor { } else if (mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; } else { - throw new ExtractionException("Unknown media format: " + mimeType); + mediaFormat = null; } - videoStreams.add(new VideoStream(recording.getString("recording_url"), - mediaFormat, recording.getInt("height") + "p")); + // Don't use the containsSimilarStream method because it will remove the + // extraction of some video versions (mostly languages) + videoStreams.add(new VideoStream.Builder() + .setId(recording.getString("filename", ID_UNKNOWN)) + .setContent(recording.getString("recording_url"), true) + .setIsVideoOnly(false) + .setMediaFormat(mediaFormat) + .setResolution(recording.getInt("height") + "p") + .build()); } } + return videoStreams; } @@ -163,7 +181,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { conferenceData = JsonParser.object() .from(downloader.get(data.getString("conference_url")).responseBody()); } catch (final JsonParserException jpe) { - throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe); + throw new ExtractionException("Could not parse json returned by URL: " + videoUrl, + jpe); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index f80815d1..bec41f48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -39,14 +40,30 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; +import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; + public class PeertubeStreamExtractor extends StreamExtractor { + private static final String ACCOUNT_HOST = "account.host"; + private static final String ACCOUNT_NAME = "account.name"; + private static final String FILES = "files"; + private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl"; + private static final String FILE_URL = "fileUrl"; + private static final String PLAYLIST_URL = "playlistUrl"; + private static final String RESOLUTION_ID = "resolution.id"; + private static final String STREAMING_PLAYLISTS = "streamingPlaylists"; + private final String baseUrl; private JsonObject json; + private final List subtitles = new ArrayList<>(); + private final List audioStreams = new ArrayList<>(); + private final List videoStreams = new ArrayList<>(); public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException { @@ -85,9 +102,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { } catch (final ParsingException e) { return Description.EMPTY_DESCRIPTION; } - if (text.length() == 250 && text.substring(247).equals("...")) { - //if description is shortened, get full description + // If description is shortened, get full description final Downloader dl = NewPipe.getDownloader(); try { final Response response = dl.get(baseUrl @@ -95,8 +111,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { + getId() + "/description"); final JsonObject jsonObject = JsonParser.object().from(response.responseBody()); text = JsonUtils.getString(jsonObject, "description"); - } catch (ReCaptchaException | IOException | JsonParserException e) { - e.printStackTrace(); + } catch (final IOException | ReCaptchaException | JsonParserException ignored) { + // Something went wrong when getting the full description, use the shortened one } } return new Description(text, Description.MARKDOWN); @@ -119,8 +135,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Override public long getTimeStamp() throws ParsingException { - final long timestamp = - getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + final long timestamp = getTimestampSeconds( + "((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); if (timestamp == -2) { // regex for timestamp was not found @@ -148,10 +164,10 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getUploaderUrl() throws ParsingException { - final String name = JsonUtils.getString(json, "account.name"); - final String host = JsonUtils.getString(json, "account.host"); - return getService().getChannelLHFactory() - .fromId("accounts/" + name + "@" + host, baseUrl).getUrl(); + final String name = JsonUtils.getString(json, ACCOUNT_NAME); + final String host = JsonUtils.getString(json, ACCOUNT_HOST); + return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl) + .getUrl(); } @Nonnull @@ -199,77 +215,51 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getHlsUrl() { - return json.getArray("streamingPlaylists").getObject(0).getString("playlistUrl"); + assertPageFetched(); + + if (getStreamType() == StreamType.VIDEO_STREAM + && !isNullOrEmpty(json.getObject(FILES))) { + return json.getObject(FILES).getString(PLAYLIST_URL, EMPTY_STRING); + } else { + return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL, + EMPTY_STRING); + } } @Override - public List getAudioStreams() { - return Collections.emptyList(); + public List getAudioStreams() throws ParsingException { + assertPageFetched(); + + /* + Some videos have audio streams, some videos don't have audio streams. + So an audio stream may be available if a video stream is available. + Audio streams are also not returned as separated streams for livestreams. + That's why the extraction of audio streams is only run when there are video streams + extracted and when the content is not a livestream. + */ + if (audioStreams.isEmpty() && videoStreams.isEmpty() + && getStreamType() == StreamType.VIDEO_STREAM) { + getStreams(); + } + + return audioStreams; } @Override public List getVideoStreams() throws ExtractionException { assertPageFetched(); - final List videoStreams = new ArrayList<>(); - // mp4 - try { - videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files"))); - } catch (final Exception ignored) { } - - // HLS - try { - final JsonArray streamingPlaylists = json.getArray("streamingPlaylists"); - for (final Object p : streamingPlaylists) { - if (!(p instanceof JsonObject)) { - continue; - } - final JsonObject playlist = (JsonObject) p; - videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files"))); + if (videoStreams.isEmpty()) { + if (getStreamType() == StreamType.VIDEO_STREAM) { + getStreams(); + } else { + extractLiveVideoStreams(); } - } catch (final Exception e) { - throw new ParsingException("Could not get video streams", e); - } - - if (getStreamType() == StreamType.LIVE_STREAM) { - videoStreams.add(new VideoStream(getHlsUrl(), MediaFormat.MPEG_4, "720p")); } return videoStreams; } - private List getVideoStreamsFromArray(final JsonArray streams) - throws ParsingException { - try { - final List videoStreams = new ArrayList<>(); - for (final Object s : streams) { - if (!(s instanceof JsonObject)) { - continue; - } - final JsonObject stream = (JsonObject) s; - final String url; - if (stream.has("fileDownloadUrl")) { - url = JsonUtils.getString(stream, "fileDownloadUrl"); - } else { - url = JsonUtils.getString(stream, "fileUrl"); - } - final String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); - final String resolution = JsonUtils.getString(stream, "resolution.label"); - final String extension = url.substring(url.lastIndexOf(".") + 1); - final MediaFormat format = MediaFormat.getFromSuffix(extension); - final VideoStream videoStream - = new VideoStream(url, torrentUrl, format, resolution); - if (!Stream.containSimilarStream(videoStream, videoStreams)) { - videoStreams.add(videoStream); - } - } - return videoStreams; - } catch (final Exception e) { - throw new ParsingException("Could not get video streams from array"); - } - - } - @Override public List getVideoOnlyStreams() { return Collections.emptyList(); @@ -284,13 +274,9 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getSubtitles(final MediaFormat format) { - final List filteredSubs = new ArrayList<>(); - for (final SubtitlesStream sub : subtitles) { - if (sub.getFormat() == format) { - filteredSubs.add(sub); - } - } - return filteredSubs; + return subtitles.stream() + .filter(sub -> sub.getFormat() == format) + .collect(Collectors.toList()); } @Override @@ -304,8 +290,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { final List tags = getTags(); final String apiUrl; if (tags.isEmpty()) { - apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name") - + "@" + JsonUtils.getString(json, "account.host") + apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, ACCOUNT_NAME) + + "@" + JsonUtils.getString(json, ACCOUNT_HOST) + "/videos?start=0&count=8"; } else { apiUrl = getRelatedItemsUrl(tags); @@ -314,7 +300,8 @@ public class PeertubeStreamExtractor extends StreamExtractor { if (Utils.isBlank(apiUrl)) { return null; } else { - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector( + getServiceId()); getStreamsFromApi(collector, apiUrl); return collector; } @@ -332,11 +319,13 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { return JsonUtils.getString(json, "support"); } catch (final ParsingException e) { - return ""; + return EMPTY_STRING; } } - private String getRelatedItemsUrl(final List tags) throws UnsupportedEncodingException { + @Nonnull + private String getRelatedItemsUrl(@Nonnull final List tags) + throws UnsupportedEncodingException { final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; final StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); @@ -348,7 +337,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { } private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl) - throws ReCaptchaException, IOException, ParsingException { + throws IOException, ReCaptchaException, ParsingException { final Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; if (response != null && !Utils.isBlank(response.responseBody())) { @@ -365,21 +354,20 @@ public class PeertubeStreamExtractor extends StreamExtractor { } private void collectStreamsFrom(final StreamInfoItemsCollector collector, - final JsonObject jsonObject) - throws ParsingException { + final JsonObject jsonObject) throws ParsingException { final JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(jsonObject, "data"); } catch (final Exception e) { - throw new ParsingException("unable to extract related videos", e); + throw new ParsingException("Could not extract related videos", e); } for (final Object c : contents) { if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; - final PeertubeStreamInfoItemExtractor extractor - = new PeertubeStreamInfoItemExtractor(item, baseUrl); - //do not add the same stream in related streams + final PeertubeStreamInfoItemExtractor extractor = + new PeertubeStreamInfoItemExtractor(item, baseUrl); + // Do not add the same stream in related streams if (!extractor.getUrl().equals(getUrl())) { collector.commit(extractor); } @@ -395,7 +383,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { if (response != null) { setInitialData(response.responseBody()); } else { - throw new ExtractionException("Unable to extract PeerTube channel data"); + throw new ExtractionException("Could not extract PeerTube channel data"); } loadSubtitles(); @@ -405,10 +393,10 @@ public class PeertubeStreamExtractor extends StreamExtractor { try { json = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { - throw new ExtractionException("Unable to extract PeerTube stream data", e); + throw new ExtractionException("Could not extract PeerTube stream data", e); } if (json == null) { - throw new ExtractionException("Unable to extract PeerTube stream data"); + throw new ExtractionException("Could not extract PeerTube stream data"); } PeertubeParsingHelper.validate(json); } @@ -429,16 +417,253 @@ public class PeertubeStreamExtractor extends StreamExtractor { final String ext = url.substring(url.lastIndexOf(".") + 1); final MediaFormat fmt = MediaFormat.getFromSuffix(ext); if (fmt != null && !isNullOrEmpty(languageCode)) { - subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + subtitles.add(new SubtitlesStream.Builder() + .setContent(url, true) + .setMediaFormat(fmt) + .setLanguageCode(languageCode) + .setAutoGenerated(false) + .build()); } } } - } catch (final Exception e) { - // ignore all exceptions + } catch (final Exception ignored) { + // Ignore all exceptions } } } + private void extractLiveVideoStreams() throws ParsingException { + try { + final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS); + for (final Object s : streamingPlaylists) { + if (!(s instanceof JsonObject)) { + continue; + } + final JsonObject stream = (JsonObject) s; + // Don't use the containsSimilarStream method because it will always return false + // so if there are multiples HLS URLs returned, only the first will be extracted in + // this case. + videoStreams.add(new VideoStream.Builder() + .setId(String.valueOf(stream.getInt("id", -1))) + .setContent(stream.getString(PLAYLIST_URL, EMPTY_STRING), true) + .setIsVideoOnly(false) + .setResolution(EMPTY_STRING) + .setMediaFormat(MediaFormat.MPEG_4) + .setDeliveryMethod(DeliveryMethod.HLS) + .build()); + } + } catch (final Exception e) { + throw new ParsingException("Could not get video streams", e); + } + } + + private void getStreams() throws ParsingException { + // Progressive streams + getStreamsFromArray(json.getArray(FILES), EMPTY_STRING); + + // HLS streams + try { + final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS); + for (final Object p : streamingPlaylists) { + if (!(p instanceof JsonObject)) { + continue; + } + final JsonObject playlist = (JsonObject) p; + final String playlistUrl = playlist.getString(PLAYLIST_URL); + getStreamsFromArray(playlist.getArray(FILES), playlistUrl); + } + } catch (final Exception e) { + throw new ParsingException("Could not get streams", e); + } + } + + private void getStreamsFromArray(@Nonnull final JsonArray streams, + final String playlistUrl) throws ParsingException { + try { + /* + Starting with version 3.4.0 of PeerTube, HLS playlist of stream resolutions contain the + UUID of the stream, so we can't use the same system to get HLS playlist URL of streams + without fetching the master playlist. + These UUIDs are the same that the ones returned into the fileUrl and fileDownloadUrl + strings. + */ + final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl) + && playlistUrl.endsWith("-master.m3u8"); + + for (final Object s : streams) { + if (!(s instanceof JsonObject)) { + continue; + } + + final JsonObject stream = (JsonObject) s; + final String resolution = JsonUtils.getString(stream, "resolution.label"); + final String url; + final String idSuffix; + + // Extract stream version of streams first + if (stream.has(FILE_URL)) { + url = JsonUtils.getString(stream, FILE_URL); + idSuffix = FILE_URL; + } else { + url = JsonUtils.getString(stream, FILE_DOWNLOAD_URL); + idSuffix = FILE_DOWNLOAD_URL; + } + + if (isNullOrEmpty(url)) { + // Not a valid stream URL + return; + } + + if (resolution.toLowerCase().contains("audio")) { + // An audio stream + addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution, + idSuffix, url, playlistUrl); + } else { + // A video stream + addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution, + idSuffix, url, playlistUrl); + } + } + } catch (final Exception e) { + throw new ParsingException("Could not get streams from array", e); + } + } + + @Nonnull + private String getHlsPlaylistUrlFromFragmentedFileUrl( + @Nonnull final JsonObject streamJsonObject, + @Nonnull final String idSuffix, + @Nonnull final String format, + @Nonnull final String url) throws ParsingException { + final String streamUrl; + if (FILE_DOWNLOAD_URL.equals(idSuffix)) { + streamUrl = JsonUtils.getString(streamJsonObject, FILE_URL); + } else { + streamUrl = url; + } + return streamUrl.replace("-fragmented." + format, ".m3u8"); + } + + @Nonnull + private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject, + @Nonnull final String playlistUrl) + throws ParsingException { + return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject, + RESOLUTION_ID).toString()); + } + + private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject, + final boolean isInstanceUsingRandomUuidsForHlsStreams, + @Nonnull final String resolution, + @Nonnull final String idSuffix, + @Nonnull final String url, + @Nullable final String playlistUrl) throws ParsingException { + final String extension = url.substring(url.lastIndexOf(".") + 1); + final MediaFormat format = MediaFormat.getFromSuffix(extension); + final String id = resolution + "-" + extension; + + // Add progressive HTTP streams first + audioStreams.add(new AudioStream.Builder() + .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP) + .setContent(url, true) + .setMediaFormat(format) + .setAverageBitrate(UNKNOWN_BITRATE) + .build()); + + // Then add HLS streams + if (!isNullOrEmpty(playlistUrl)) { + final String hlsStreamUrl; + if (isInstanceUsingRandomUuidsForHlsStreams) { + hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, + extension, url); + + } else { + hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl); + } + final AudioStream audioStream = new AudioStream.Builder() + .setId(id + "-" + DeliveryMethod.HLS) + .setContent(hlsStreamUrl, true) + .setDeliveryMethod(DeliveryMethod.HLS) + .setMediaFormat(format) + .setAverageBitrate(UNKNOWN_BITRATE) + .setBaseUrl(playlistUrl) + .build(); + if (!Stream.containSimilarStream(audioStream, audioStreams)) { + audioStreams.add(audioStream); + } + } + + // Add finally torrent URLs + final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl"); + if (!isNullOrEmpty(torrentUrl)) { + audioStreams.add(new AudioStream.Builder() + .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT) + .setContent(torrentUrl, true) + .setDeliveryMethod(DeliveryMethod.TORRENT) + .setMediaFormat(format) + .setAverageBitrate(UNKNOWN_BITRATE) + .build()); + } + } + + private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject, + final boolean isInstanceUsingRandomUuidsForHlsStreams, + @Nonnull final String resolution, + @Nonnull final String idSuffix, + @Nonnull final String url, + @Nullable final String playlistUrl) throws ParsingException { + final String extension = url.substring(url.lastIndexOf(".") + 1); + final MediaFormat format = MediaFormat.getFromSuffix(extension); + final String id = resolution + "-" + extension; + + // Add progressive HTTP streams first + videoStreams.add(new VideoStream.Builder() + .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP) + .setContent(url, true) + .setIsVideoOnly(false) + .setResolution(resolution) + .setMediaFormat(format) + .build()); + + // Then add HLS streams + if (!isNullOrEmpty(playlistUrl)) { + final String hlsStreamUrl; + if (isInstanceUsingRandomUuidsForHlsStreams) { + hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, + extension, url); + } else { + hlsStreamUrl = playlistUrl.replace("master", JsonUtils.getNumber( + streamJsonObject, RESOLUTION_ID).toString()); + } + + final VideoStream videoStream = new VideoStream.Builder() + .setId(id + "-" + DeliveryMethod.HLS) + .setContent(hlsStreamUrl, true) + .setIsVideoOnly(false) + .setDeliveryMethod(DeliveryMethod.HLS) + .setResolution(resolution) + .setMediaFormat(format) + .setBaseUrl(playlistUrl) + .build(); + if (!Stream.containSimilarStream(videoStream, videoStreams)) { + videoStreams.add(videoStream); + } + } + + // Add finally torrent URLs + final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl"); + if (!isNullOrEmpty(torrentUrl)) { + videoStreams.add(new VideoStream.Builder() + .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT) + .setContent(torrentUrl, true) + .setIsVideoOnly(false) + .setDeliveryMethod(DeliveryMethod.TORRENT) + .setResolution(resolution) + .setMediaFormat(format) + .build()); + } + } + @Nonnull @Override public String getName() throws ParsingException { @@ -448,7 +673,7 @@ public class PeertubeStreamExtractor extends StreamExtractor { @Nonnull @Override public String getHost() throws ParsingException { - return JsonUtils.getString(json, "account.host"); + return JsonUtils.getString(json, ACCOUNT_HOST); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 24bb6ec5..160bf572 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -1,6 +1,9 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId; +import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; +import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; @@ -26,6 +29,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; @@ -58,13 +62,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor { final String policy = track.getString("policy", EMPTY_STRING); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { isAvailable = false; + if (policy.equals("SNIP")) { throw new SoundCloudGoPlusContentException(); } + if (policy.equals("BLOCK")) { throw new GeographicRestrictionException( "This track is not available in user's country"); } + throw new ContentNotAvailableException("Content not available: policy " + policy); } } @@ -72,7 +79,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { @Nonnull @Override public String getId() { - return track.getInt("id") + EMPTY_STRING; + return String.valueOf(track.getInt("id")); } @Nonnull @@ -162,17 +169,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { // Streams can be streamable and downloadable - or explicitly not. // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. + // If audio streams were calculated, return the calculated result if (!track.getBoolean("streamable") || !isAvailable) { return audioStreams; } try { final JsonArray transcodings = track.getObject("media").getArray("transcodings"); - if (transcodings != null) { + if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings), audioStreams); } + extractDownloadableFileIfAvailable(audioStreams); } catch (final NullPointerException e) { throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e); } @@ -180,7 +189,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return audioStreams; } - private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) { + private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) { boolean presence = false; for (final Object transcoding : transcodings) { final JsonObject transcodingJsonObject = (JsonObject) transcoding; @@ -195,48 +204,68 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Nonnull - private static String getTranscodingUrl(final String endpointUrl, - final String protocol) + private String getTranscodingUrl(final String endpointUrl, + final String protocol) throws IOException, ExtractionException { final Downloader downloader = NewPipe.getDownloader(); final String apiStreamUrl = endpointUrl + "?client_id=" - + SoundcloudParsingHelper.clientId(); + + clientId(); final String response = downloader.get(apiStreamUrl).responseBody(); final JsonObject urlObject; try { urlObject = JsonParser.object().from(response); } catch (final JsonParserException e) { - throw new ParsingException("Could not parse streamable url", e); + throw new ParsingException("Could not parse streamable URL", e); } + final String urlString = urlObject.getString("url"); if (protocol.equals("progressive")) { return urlString; } else if (protocol.equals("hls")) { - try { - return getSingleUrlFromHlsManifest(urlString); - } catch (final ParsingException ignored) { - } + return getSingleUrlFromHlsManifest(urlString); } + // else, unknown protocol - return ""; + return EMPTY_STRING; } - private static void extractAudioStreams(final JsonArray transcodings, - final boolean mp3ProgressiveInStreams, - final List audioStreams) { + @Nullable + private String getDownloadUrl(@Nonnull final String trackId) + throws IOException, ExtractionException { + final Downloader dl = NewPipe.getDownloader(); + final JsonObject downloadJsonObject; + + final String response = dl.get(SOUNDCLOUD_API_V2_URL + "tracks/" + trackId + + "/download" + "?client_id=" + clientId()).responseBody(); + try { + downloadJsonObject = JsonParser.object().from(response); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse download URL", e); + } + final String redirectUri = downloadJsonObject.getString("redirectUri"); + if (!isNullOrEmpty(redirectUri)) { + return redirectUri; + } + return null; + } + + private void extractAudioStreams(@Nonnull final JsonArray transcodings, + final boolean mp3ProgressiveInStreams, + final List audioStreams) { for (final Object transcoding : transcodings) { final JsonObject transcodingJsonObject = (JsonObject) transcoding; final String url = transcodingJsonObject.getString("url"); if (isNullOrEmpty(url)) { continue; } + final String mediaUrl; - final String preset = transcodingJsonObject.getString("preset"); + final String preset = transcodingJsonObject.getString("preset", ID_UNKNOWN); final String protocol = transcodingJsonObject.getObject("format") .getString("protocol"); MediaFormat mediaFormat = null; - int bitrate = 0; + int averageBitrate = UNKNOWN_BITRATE; if (preset.contains("mp3")) { // Don't add the MP3 HLS stream if there is a progressive stream present // because the two have the same bitrate @@ -244,36 +273,75 @@ public class SoundcloudStreamExtractor extends StreamExtractor { continue; } mediaFormat = MediaFormat.MP3; - bitrate = 128; + averageBitrate = 128; } else if (preset.contains("opus")) { mediaFormat = MediaFormat.OPUS; - bitrate = 64; + averageBitrate = 64; } - if (mediaFormat != null) { - try { - mediaUrl = getTranscodingUrl(url, protocol); - if (!mediaUrl.isEmpty()) { - audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate)); + try { + mediaUrl = getTranscodingUrl(url, protocol); + if (!mediaUrl.isEmpty()) { + final AudioStream audioStream = new AudioStream.Builder() + .setId(preset) + .setContent(mediaUrl, true) + .setMediaFormat(mediaFormat) + .setAverageBitrate(averageBitrate) + .build(); + if (!Stream.containSimilarStream(audioStream, audioStreams)) { + audioStreams.add(audioStream); } - } catch (final Exception ignored) { - // something went wrong when parsing this transcoding, don't add it to - // audioStreams } + } catch (final Exception ignored) { + // Something went wrong when parsing this transcoding, don't add it to the + // audioStreams + } + } + } + + /** + * Add the downloadable format if it is available. + * + *

+ * A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean + * we can download it: if the value of the {@code has_download_left} boolean is true, the track + * can be downloaded; otherwise not. + *

+ * + * @param audioStreams the audio streams to which add the downloadable file + */ + public void extractDownloadableFileIfAvailable(final List audioStreams) { + if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) { + try { + final String downloadUrl = getDownloadUrl(getId()); + if (!isNullOrEmpty(downloadUrl)) { + audioStreams.add(new AudioStream.Builder() + .setId("original-format") + .setContent(downloadUrl, true) + .setAverageBitrate(UNKNOWN_BITRATE) + .build()); + } + } catch (final Exception ignored) { + // If something went wrong when trying to get the download URL, ignore the + // exception throw because this "stream" is not necessary to play the track } } } /** * Parses a SoundCloud HLS manifest to get a single URL of HLS streams. + * *

* This method downloads the provided manifest URL, find all web occurrences in the manifest, * get the last segment URL, changes its segment range to {@code 0/track-length} and return * this string. + *

+ * * @param hlsManifestUrl the URL of the manifest to be parsed * @return a single URL that contains a range equal to the length of the track */ - private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl) + @Nonnull + private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl) throws ParsingException { final Downloader dl = NewPipe.getDownloader(); final String hlsManifestResponse; @@ -326,7 +394,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId()) - + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId()); + + "/related?client_id=" + urlEncode(clientId()); SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); return collector; @@ -355,19 +423,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor { // Tags are separated by spaces, but they can be multiple words escaped by quotes " final String[] tagList = track.getString("tag_list").split(" "); final List tags = new ArrayList<>(); - String escapedTag = ""; + final StringBuilder escapedTag = new StringBuilder(); boolean isEscaped = false; for (final String tag : tagList) { if (tag.startsWith("\"")) { - escapedTag += tag.replace("\"", ""); + escapedTag.append(tag.replace("\"", "")); isEscaped = true; } else if (isEscaped) { if (tag.endsWith("\"")) { - escapedTag += " " + tag.replace("\"", ""); + escapedTag.append(" ").append(tag.replace("\"", "")); isEscaped = false; - tags.add(escapedTag); + tags.add(escapedTag.toString()); } else { - escapedTag += " " + tag; + escapedTag.append(" ").append(tag); } } else if (!tag.isEmpty()) { tags.add(tag);