From 6031695343b9705013e89cf954b6edbab07552e6 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Wed, 22 Nov 2017 18:39:38 +0100 Subject: [PATCH 1/8] Addede fetching subtitles for youtube and its tests --- NewPipeExtractor.iml | 25 +------ .../soundcloud/SoundcloudStreamExtractor.java | 6 ++ .../youtube/YoutubeStreamExtractor.java | 73 +++++++++++++++++-- .../extractor/stream/StreamExtractor.java | 3 + .../SoundcloudStreamExtractorDefaultTest.java | 6 ++ .../YoutubeStreamExtractorDefaultTest.java | 7 ++ .../YoutubeStreamExtractorRestrictedTest.java | 6 ++ 7 files changed, 100 insertions(+), 26 deletions(-) diff --git a/NewPipeExtractor.iml b/NewPipeExtractor.iml index fa027b00..cdb9deeb 100644 --- a/NewPipeExtractor.iml +++ b/NewPipeExtractor.iml @@ -1,30 +1,13 @@ - - - - - - - - - - + + - - - - + + - - - - - \ No newline at end of file diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index ba2cd2d1..aa6e732e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; public class SoundcloudStreamExtractor extends StreamExtractor { @@ -192,6 +193,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return null; } + @Override + public HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException { + return new HashMap<>(); + } + @Override public StreamType getStreamType() { return StreamType.AUDIO_STREAM; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 8dbb9ead..2a030f59 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -1,7 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -20,10 +22,7 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -421,6 +420,59 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoOnlyStreams; } + /** + * Example output: + * { + * #language code#: [ + * [0]"captions URL", + * [1]"language Name" + * ], + * "a.en": { // a.#language code# == auto generated + * [0]"https://youtube.com/api/timedtext..." + * [1]"English (Auto-generated)" + * }, + * ".en": { // .#language code# == normal (not auto generated) + * [0]"https://youtube.com/api/timedtext..." + * [1]"English" + * } + * } + * + * Example usage: + * 1) Get list of keys in the Map if there are any + * 2) Get + * + * @return Map(String, StringArray[2]) + * @throws IOException - Thrown when parsing HTML page + * @throws ExtractionException - Thrown when parsing HTML + * @throws JsonParserException - Thrown when parsing JSON from the web page + */ + @Override + public HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException { + HashMap result = new HashMap<>(); + + JsonObject playerConfig = getPlayerConfig(getPageHtml()); + + String playerResponse = playerConfig.getObject("args").getString("player_response"); + + if (!JsonParser.object().from(playerResponse).has("captions")) { + return new HashMap<>(); + } + + JsonObject captions = JsonParser.object().from(playerResponse).getObject("captions"); + JsonArray captionsArray = captions.getObject("playerCaptionsTracklistRenderer").getArray("captionTracks"); + + for (int x = 0; x < captionsArray.size(); x++) { + String captionsUrlAndName[] = new String[2]; + captionsUrlAndName[0] = captionsArray.getObject(x).getString("baseUrl"); + captionsUrlAndName[1] = captionsArray.getObject(x).getObject("name").getString("simpleText"); + String captionsLangCode = captionsArray.getObject(x).getString("vssId"); + + result.put(captionsLangCode, captionsUrlAndName); + } + + return result; + } + @Override public StreamType getStreamType() throws ParsingException { //todo: if implementing livestream support this value should be generated dynamically @@ -498,13 +550,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static volatile String decryptionCode = ""; + private static String pageHtml = null; + + private String getPageHtml() throws IOException, ExtractionException{ + if (pageHtml == null) { + Downloader dl = NewPipe.getDownloader(); + pageHtml = dl.download(getCleanUrl()); + } + return pageHtml; + } + @Override public void fetchPage() throws IOException, ExtractionException { Downloader dl = NewPipe.getDownloader(); - String pageContent = dl.download(getCleanUrl()); + String pageContent = getPageHtml(); doc = Jsoup.parse(pageContent, getCleanUrl()); + String playerUrl; // Check if the video is age restricted if (pageContent.contains(". */ +import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; @@ -27,6 +28,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.IOException; +import java.util.HashMap; import java.util.List; /** @@ -64,6 +66,7 @@ public abstract class StreamExtractor extends Extractor { public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; + public abstract HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException; public abstract StreamType getStreamType() throws ParsingException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 2d76bdd3..6df8d430 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud; +import com.grack.nanojson.JsonParserException; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; @@ -101,4 +102,9 @@ public class SoundcloudStreamExtractorDefaultTest { assertFalse(relatedVideos.getItemList().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty()); } + + @Test + public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitles() != null); + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 63dfaed0..d05695f7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import com.grack.nanojson.JsonParserException; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; @@ -12,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; +import java.util.HashMap; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -148,4 +150,9 @@ public class YoutubeStreamExtractorDefaultTest { } assertTrue(relatedVideos.getErrors().isEmpty()); } + + @Test + public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitles() != null); + } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index fadd13df..f9bc3999 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; +import com.grack.nanojson.JsonParserException; import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; @@ -103,4 +104,9 @@ public class YoutubeStreamExtractorRestrictedTest { 0 <= s.format && s.format <= 4); } } + + @Test + public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitles() != null); + } } From 6e3651fdf52cb5053ad5367ebcce6be9040b91ca Mon Sep 17 00:00:00 2001 From: tonakriz Date: Thu, 23 Nov 2017 00:10:12 +0100 Subject: [PATCH 2/8] Added method to download subtitles as VTT --- .../soundcloud/SoundcloudStreamExtractor.java | 8 ++++- .../youtube/YoutubeStreamExtractor.java | 36 +++++-------------- .../extractor/stream/StreamExtractor.java | 4 ++- .../newpipe/extractor/utils/Parser.java | 10 ++---- .../SoundcloudStreamExtractorDefaultTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 15 ++++++-- .../YoutubeStreamExtractorRestrictedTest.java | 9 +++-- 7 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index aa6e732e..28bc4bdb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.Parser; @@ -194,10 +195,15 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException { + public HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException { return new HashMap<>(); } + @Override + public String downloadSubtitles(String URL) throws IOException, ReCaptchaException { + return ""; + } + @Override public StreamType getStreamType() { return StreamType.AUDIO_STREAM; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 2a030f59..4fa3b7af 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -420,34 +420,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoOnlyStreams; } - /** - * Example output: - * { - * #language code#: [ - * [0]"captions URL", - * [1]"language Name" - * ], - * "a.en": { // a.#language code# == auto generated - * [0]"https://youtube.com/api/timedtext..." - * [1]"English (Auto-generated)" - * }, - * ".en": { // .#language code# == normal (not auto generated) - * [0]"https://youtube.com/api/timedtext..." - * [1]"English" - * } - * } - * - * Example usage: - * 1) Get list of keys in the Map if there are any - * 2) Get - * - * @return Map(String, StringArray[2]) - * @throws IOException - Thrown when parsing HTML page - * @throws ExtractionException - Thrown when parsing HTML - * @throws JsonParserException - Thrown when parsing JSON from the web page - */ @Override - public HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException { + public HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException { HashMap result = new HashMap<>(); JsonObject playerConfig = getPlayerConfig(getPageHtml()); @@ -473,6 +447,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { return result; } + @Override + public String downloadSubtitles(String URL) throws IOException, ReCaptchaException { + Downloader dl = NewPipe.getDownloader(); + // Instead of the WebVTT 'vtt' we can use also Timed Text Markup Language 'ttml' + String URLasVTT = URL.replaceAll("&fmt=[^&]*", "&fmt=vtt"); + return dl.download(URLasVTT); + } + @Override public StreamType getStreamType() throws ParsingException { //todo: if implementing livestream support this value should be generated dynamically diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index fa7a3d6a..fc49e653 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -26,6 +26,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import java.io.IOException; import java.util.HashMap; @@ -66,7 +67,8 @@ public abstract class StreamExtractor extends Extractor { public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - public abstract HashMap getSubtitles() throws IOException, ExtractionException, JsonParserException; + public abstract HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException; + public abstract String downloadSubtitles(String URL) throws IOException, ReCaptchaException; public abstract StreamType getStreamType() throws ParsingException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; diff --git a/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 2db76246..9920c7fb 100644 --- a/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -54,7 +54,6 @@ public class Parser { if (foundMatch) { return mat.group(group); } else { - //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); if (input.length() > 1024) { throw new RegexException("failed to find pattern \"" + pattern); } else { @@ -64,12 +63,9 @@ public class Parser { } public static boolean isMatch(String pattern, String input) { - try { - matchGroup1(pattern, input); - return true; - } catch (RegexException e) { - return false; - } + Pattern pat = Pattern.compile(pattern); + Matcher mat = pat.matcher(input); + return mat.find(); } public static Map compatParseMap(final String input) throws UnsupportedEncodingException { diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 6df8d430..882e0da3 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -105,6 +105,6 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() != null); + assertTrue(extractor.getSubtitlesList() != null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index d05695f7..77d484a2 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -152,7 +152,18 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() != null); + public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitlesList() != null); + } + + @Test + public void testDownloadSubtitles() throws Exception { + try { + extractor.downloadSubtitles(extractor.getSubtitlesList().get("en")[0]); + // Video has no subtitles! + assert false; + } catch (Exception e) { + assert true; + } } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index f9bc3999..7b604f43 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -106,7 +106,12 @@ public class YoutubeStreamExtractorRestrictedTest { } @Test - public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() != null); + public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitlesList() != null); + } + + @Test + public void testDownloadSubtitles() throws Exception { + assertTrue(extractor.downloadSubtitles("https://youtu.be/FmG385_uUys?t=174") != null); } } From 72f9ac223e25e246218e5eb34040077161831561 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Thu, 23 Nov 2017 11:47:05 +0100 Subject: [PATCH 3/8] Using Class instead of HashMap, removed downloadSubtitles method --- .../schabi/newpipe/extractor/Subtitles.java | 31 ++++++++++++++ .../soundcloud/SoundcloudStreamExtractor.java | 14 ++----- .../youtube/YoutubeStreamExtractor.java | 40 +++++++++---------- .../extractor/stream/StreamExtractor.java | 5 +-- 4 files changed, 54 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/Subtitles.java diff --git a/src/main/java/org/schabi/newpipe/extractor/Subtitles.java b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java new file mode 100644 index 00000000..12e1dcbc --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor; + +public class Subtitles { + private String languageName; + private String languageCode; + private String URL; + private boolean autoGenerated; + + public Subtitles(String languageName, String languageCode, String URL, boolean autoGenerated) { + this.languageName = languageName; + this.languageCode = languageCode; + this.URL = URL; + this.autoGenerated = autoGenerated; + } + + public String getLanguageName() { + return languageName; + } + + public String getLanguageCode() { + return languageCode; + } + + public String getURL() { + return URL; + } + + public boolean isAutoGenerated() { + return autoGenerated; + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 28bc4bdb..d8b09a11 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -3,10 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -195,13 +192,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException { - return new HashMap<>(); - } - - @Override - public String downloadSubtitles(String URL) throws IOException, ReCaptchaException { - return ""; + public Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException { + return null; } @Override diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 4fa3b7af..b9345e2e 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -13,6 +13,7 @@ import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.Subtitles; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -421,40 +422,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException { - HashMap result = new HashMap<>(); - + public Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException { JsonObject playerConfig = getPlayerConfig(getPageHtml()); - String playerResponse = playerConfig.getObject("args").getString("player_response"); - if (!JsonParser.object().from(playerResponse).has("captions")) { - return new HashMap<>(); - } + if (!JsonParser.object().from(playerResponse).has("captions")) return null; JsonObject captions = JsonParser.object().from(playerResponse).getObject("captions"); JsonArray captionsArray = captions.getObject("playerCaptionsTracklistRenderer").getArray("captionTracks"); - for (int x = 0; x < captionsArray.size(); x++) { - String captionsUrlAndName[] = new String[2]; - captionsUrlAndName[0] = captionsArray.getObject(x).getString("baseUrl"); - captionsUrlAndName[1] = captionsArray.getObject(x).getObject("name").getString("simpleText"); + int captionsSize = captionsArray.size(); + Subtitles[] result = new Subtitles[captionsSize]; + + for (int x = 0; x < captionsSize; x++) { + String baseUrl = captionsArray.getObject(x).getString("baseUrl"); + + String languageName = captionsArray.getObject(x).getObject("name").getString("simpleText"); + String URL = baseUrl.replaceAll("&fmt=[^&]*", "&fmt=vtt"); + String captionsLangCode = captionsArray.getObject(x).getString("vssId"); + String languageCode = captionsLangCode.replaceAll("(a\\.)|(\\.)", ""); - result.put(captionsLangCode, captionsUrlAndName); + boolean isAutoGenerated = captionsLangCode.startsWith("a."); + + result[x] = new Subtitles(languageName, languageCode, URL, isAutoGenerated); } - + return result; } - @Override - public String downloadSubtitles(String URL) throws IOException, ReCaptchaException { - Downloader dl = NewPipe.getDownloader(); - // Instead of the WebVTT 'vtt' we can use also Timed Text Markup Language 'ttml' - String URLasVTT = URL.replaceAll("&fmt=[^&]*", "&fmt=vtt"); - return dl.download(URLasVTT); - } - @Override public StreamType getStreamType() throws ParsingException { //todo: if implementing livestream support this value should be generated dynamically @@ -703,7 +699,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException { Context context = Context.enter(); context.setOptimizationLevel(-1); - Object result = null; + Object result; try { ScriptableObject scope = context.initStandardObjects(); context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index fc49e653..7030f4aa 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -23,13 +23,13 @@ package org.schabi.newpipe.extractor.stream; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.Subtitles; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import java.io.IOException; -import java.util.HashMap; import java.util.List; /** @@ -67,8 +67,7 @@ public abstract class StreamExtractor extends Extractor { public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - public abstract HashMap getSubtitlesList() throws IOException, ExtractionException, JsonParserException; - public abstract String downloadSubtitles(String URL) throws IOException, ReCaptchaException; + public abstract Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException; public abstract StreamType getStreamType() throws ParsingException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; From f09b4e68d00dfdffd8dfd2945457c1f7a428c524 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Thu, 23 Nov 2017 11:47:08 +0100 Subject: [PATCH 4/8] Using Class instead of HashMap, removed downloadSubtitles method --- .../services/youtube/YoutubeStreamExtractor.java | 7 +++++-- .../SoundcloudStreamExtractorDefaultTest.java | 2 +- .../youtube/YoutubeStreamExtractorDefaultTest.java | 14 ++------------ .../YoutubeStreamExtractorRestrictedTest.java | 7 +------ 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index b9345e2e..daf5b8ac 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -426,14 +426,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { JsonObject playerConfig = getPlayerConfig(getPageHtml()); String playerResponse = playerConfig.getObject("args").getString("player_response"); + // Captions does not exist, return null if (!JsonParser.object().from(playerResponse).has("captions")) return null; JsonObject captions = JsonParser.object().from(playerResponse).getObject("captions"); JsonArray captionsArray = captions.getObject("playerCaptionsTracklistRenderer").getArray("captionTracks"); int captionsSize = captionsArray.size(); - Subtitles[] result = new Subtitles[captionsSize]; + // Should not happen, if there is the "captions" object, it should always has some captions in it + if(captionsSize == 0) return null; + Subtitles[] result = new Subtitles[captionsSize]; for (int x = 0; x < captionsSize; x++) { String baseUrl = captionsArray.getObject(x).getString("baseUrl"); @@ -447,7 +450,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { result[x] = new Subtitles(languageName, languageCode, URL, isAutoGenerated); } - + return result; } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 882e0da3..6df8d430 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -105,6 +105,6 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitlesList() != null); + assertTrue(extractor.getSubtitles() != null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 77d484a2..07cad201 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -153,17 +153,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitlesList() != null); - } - - @Test - public void testDownloadSubtitles() throws Exception { - try { - extractor.downloadSubtitles(extractor.getSubtitlesList().get("en")[0]); - // Video has no subtitles! - assert false; - } catch (Exception e) { - assert true; - } + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitles() != null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index 7b604f43..9bc89eef 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -107,11 +107,6 @@ public class YoutubeStreamExtractorRestrictedTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitlesList() != null); - } - - @Test - public void testDownloadSubtitles() throws Exception { - assertTrue(extractor.downloadSubtitles("https://youtu.be/FmG385_uUys?t=174") != null); + assertTrue(extractor.getSubtitles() != null); } } From 320c0d8c99bf3d51e6748ab9cbb118ddd01389b2 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Thu, 23 Nov 2017 12:30:19 +0100 Subject: [PATCH 5/8] Fixed typos in tests --- .../soundcloud/SoundcloudStreamExtractorDefaultTest.java | 2 +- .../services/youtube/YoutubeStreamExtractorDefaultTest.java | 2 +- .../youtube/YoutubeStreamExtractorRestrictedTest.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 6df8d430..fad575e7 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -105,6 +105,6 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() != null); + assertTrue(extractor.getSubtitles() == null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 07cad201..e6459f37 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -154,6 +154,6 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles() != null); + assertTrue(extractor.getSubtitles() == null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index 9bc89eef..587bb997 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -106,7 +106,7 @@ public class YoutubeStreamExtractorRestrictedTest { } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() != null); + public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { + assertTrue(extractor.getSubtitles() == null); } } From 310b34558bbe5b7290c2149479dacd2bde576374 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Thu, 23 Nov 2017 16:33:03 +0100 Subject: [PATCH 6/8] Changed the way of getting subtitles data type, removed language name from Subtitles class --- .../schabi/newpipe/extractor/Subtitles.java | 15 +++++------ .../soundcloud/SoundcloudStreamExtractor.java | 11 +++++--- .../youtube/YoutubeStreamExtractor.java | 18 ++++++++----- .../extractor/stream/StreamExtractor.java | 4 ++- .../extractor/stream/SubtitlesFormat.java | 26 +++++++++++++++++++ .../SoundcloudStreamExtractorDefaultTest.java | 14 ++++++++-- .../YoutubeStreamExtractorDefaultTest.java | 13 ++++++---- .../YoutubeStreamExtractorRestrictedTest.java | 13 ++++++++-- 8 files changed, 85 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java diff --git a/src/main/java/org/schabi/newpipe/extractor/Subtitles.java b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java index 12e1dcbc..0542b385 100644 --- a/src/main/java/org/schabi/newpipe/extractor/Subtitles.java +++ b/src/main/java/org/schabi/newpipe/extractor/Subtitles.java @@ -1,21 +1,20 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; + public class Subtitles { - private String languageName; - private String languageCode; - private String URL; + private SubtitlesFormat format; + private String languageCode, URL; private boolean autoGenerated; - public Subtitles(String languageName, String languageCode, String URL, boolean autoGenerated) { - this.languageName = languageName; + public Subtitles(SubtitlesFormat format, String languageCode, String URL, boolean autoGenerated) { + this.format = format; this.languageCode = languageCode; this.URL = URL; this.autoGenerated = autoGenerated; } - public String getLanguageName() { - return languageName; - } + public SubtitlesFormat getFileType() { return format; } public String getLanguageCode() { return languageCode; diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index caff6062..b4702c64 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -12,9 +12,7 @@ import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; public class SoundcloudStreamExtractor extends StreamExtractor { private JsonObject track; @@ -150,7 +148,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException { + public List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException { + return null; + } + + @Override + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException { return null; } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index e246a50e..00ee2433 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -380,7 +380,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException { + public List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException { + return getSubtitles(SubtitlesFormat.TTML); + } + + @Override + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException { JsonObject playerConfig = getPlayerConfig(getPageHtml()); String playerResponse = playerConfig.getObject("args").getString("player_response"); @@ -394,19 +399,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Should not happen, if there is the "captions" object, it should always has some captions in it if(captionsSize == 0) return null; - Subtitles[] result = new Subtitles[captionsSize]; + List result = new ArrayList<>(); for (int x = 0; x < captionsSize; x++) { String baseUrl = captionsArray.getObject(x).getString("baseUrl"); - String languageName = captionsArray.getObject(x).getObject("name").getString("simpleText"); - String URL = baseUrl.replaceAll("&fmt=[^&]*", "&fmt=vtt"); + String extension = format.getExtension(); + String URL = baseUrl.replaceAll("&fmt=[^&]*", "&fmt=" + extension); String captionsLangCode = captionsArray.getObject(x).getString("vssId"); - String languageCode = captionsLangCode.replaceAll("(a\\.)|(\\.)", ""); - boolean isAutoGenerated = captionsLangCode.startsWith("a."); + String languageCode = captionsLangCode.replaceFirst((isAutoGenerated) ? "a." : ".", ""); - result[x] = new Subtitles(languageName, languageCode, URL, isAutoGenerated); + result.add(new Subtitles(format, languageCode, URL, isAutoGenerated)); } return result; diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index bf048d72..a7c38da9 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -112,7 +112,9 @@ public abstract class StreamExtractor extends Extractor { public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - public abstract Subtitles[] getSubtitles() throws IOException, ExtractionException, JsonParserException; + public abstract List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException; + + public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException; public abstract StreamType getStreamType() throws ParsingException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java b/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java new file mode 100644 index 00000000..d0a0482b --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java @@ -0,0 +1,26 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.Subtitles; + +public enum SubtitlesFormat { + // YouTube subtitles formats + // TRANSCRIPT(3) is default YT format based on TTML, + // but unlike VTT or TTML, it is NOT W3 standard + VTT (0x0, "vtt"), + TTML (0x1, "ttml"), + TRANSCRIPT1 (0x2, "srv1"), + TRANSCRIPT2 (0x3, "srv2"), + TRANSCRIPT3 (0x4, "srv3"); + + private int id; + private String extension; + + SubtitlesFormat(int id, String extension) { + this.id = id; + this.extension = extension; + } + + public String getExtension() { + return extension; + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index fad575e7..9cf1c24b 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -5,13 +5,16 @@ import org.junit.Before; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Subtitles; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import java.io.IOException; +import java.util.List; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; @@ -104,7 +107,14 @@ public class SoundcloudStreamExtractorDefaultTest { } @Test - public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() == null); + public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault() == null); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index e6459f37..fbc4d68d 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -7,10 +7,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream.StreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.*; import java.io.IOException; import java.util.HashMap; @@ -151,9 +148,15 @@ public class YoutubeStreamExtractorDefaultTest { assertTrue(relatedVideos.getErrors().isEmpty()); } + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault() == null); + } + @Test public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles() == null); + assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index 587bb997..b0f1c30f 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; @@ -105,8 +106,16 @@ public class YoutubeStreamExtractorRestrictedTest { } } + @Test - public void testGetSubtitles() throws IOException, ExtractionException, JsonParserException { - assertTrue(extractor.getSubtitles() == null); + public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault() == null); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } } From 2c7acc74f55f834aab43da362481dc455869779e Mon Sep 17 00:00:00 2001 From: tonakriz Date: Fri, 24 Nov 2017 13:57:54 +0100 Subject: [PATCH 7/8] Fixed exceptions as requested --- .../soundcloud/SoundcloudStreamExtractor.java | 4 ++-- .../services/youtube/YoutubeStreamExtractor.java | 16 +++++++++++----- .../extractor/stream/StreamExtractor.java | 5 ++--- .../SoundcloudStreamExtractorDefaultTest.java | 4 ++-- .../YoutubeStreamExtractorDefaultTest.java | 4 ++-- .../YoutubeStreamExtractorRestrictedTest.java | 4 ++-- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index b4702c64..90404282 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -148,12 +148,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor { } @Override - public List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException { + public List getSubtitlesDefault() throws IOException, ExtractionException { return null; } @Override - public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException { + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { return null; } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 00ee2433..40e10226 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -380,19 +380,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException { + public List getSubtitlesDefault() throws IOException, ExtractionException { return getSubtitles(SubtitlesFormat.TTML); } @Override - public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException { + public List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { JsonObject playerConfig = getPlayerConfig(getPageHtml()); String playerResponse = playerConfig.getObject("args").getString("player_response"); - // Captions does not exist, return null - if (!JsonParser.object().from(playerResponse).has("captions")) return null; + JsonObject captions; + try { + // Captions does not exist, return null + if (!JsonParser.object().from(playerResponse).has("captions")) return null; - JsonObject captions = JsonParser.object().from(playerResponse).getObject("captions"); + captions = JsonParser.object().from(playerResponse).getObject("captions"); + } catch (JsonParserException e) { + // Failed to parse subtitles + return null; + } JsonArray captionsArray = captions.getObject("playerCaptionsTracklistRenderer").getArray("captionTracks"); int captionsSize = captionsArray.size(); diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a7c38da9..55bb2037 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -112,9 +112,8 @@ public abstract class StreamExtractor extends Extractor { public abstract List getAudioStreams() throws IOException, ExtractionException; public abstract List getVideoStreams() throws IOException, ExtractionException; public abstract List getVideoOnlyStreams() throws IOException, ExtractionException; - public abstract List getSubtitlesDefault() throws IOException, ExtractionException, JsonParserException; - - public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException, JsonParserException; + public abstract List getSubtitlesDefault() throws IOException, ExtractionException; + public abstract List getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException; public abstract StreamType getStreamType() throws ParsingException; public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException; diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 9cf1c24b..fe23298b 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -107,13 +107,13 @@ public class SoundcloudStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault() == null); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index fbc4d68d..6e15b499 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -149,13 +149,13 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault() == null); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java index b0f1c30f..65a11ebe 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -108,13 +108,13 @@ public class YoutubeStreamExtractorRestrictedTest { @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault() == null); } @Test - public void testGetSubtitlesList() throws IOException, ExtractionException, JsonParserException { + public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(SubtitlesFormat.VTT) == null); } From bd1914612f482a5a877bcd6c71ac6873ff595a96 Mon Sep 17 00:00:00 2001 From: tonakriz Date: Fri, 24 Nov 2017 13:59:52 +0100 Subject: [PATCH 8/8] Added comment --- .../org/schabi/newpipe/extractor/stream/SubtitlesFormat.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java b/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java index d0a0482b..a4545480 100644 --- a/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java +++ b/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java @@ -6,6 +6,7 @@ public enum SubtitlesFormat { // YouTube subtitles formats // TRANSCRIPT(3) is default YT format based on TTML, // but unlike VTT or TTML, it is NOT W3 standard + // TRANSCRIPT subtitles are NOT supported by ExoPlayer, only VTT and TTML VTT (0x0, "vtt"), TTML (0x1, "ttml"), TRANSCRIPT1 (0x2, "srv1"),