Merge branch 'dev' into yt-webm-opus
This commit is contained in:
commit
fbb9a86c9f
4 changed files with 137 additions and 25 deletions
|
@ -82,6 +82,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
@Nonnull
|
||||
@Override
|
||||
public String getId() throws ParsingException {
|
||||
try {
|
||||
return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
|
||||
} catch (Exception ignored) {}
|
||||
|
||||
// fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
|
||||
try {
|
||||
Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
|
||||
if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
|
||||
|
@ -135,11 +140,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
|||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
final String el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]")
|
||||
.first().attr("title");
|
||||
|
||||
final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||
if (el != null) {
|
||||
String elTitle = el.attr("title");
|
||||
try {
|
||||
return Utils.mixedNumberWordToLong(el);
|
||||
return Utils.mixedNumberWordToLong(elTitle);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new ParsingException("Could not get subscriber count", e);
|
||||
}
|
||||
|
|
|
@ -56,19 +56,25 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
|||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
|
||||
.attr("abs:data-href");
|
||||
try {
|
||||
String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
|
||||
.attr("abs:data-href");
|
||||
|
||||
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
|
||||
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
|
||||
Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
|
||||
Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
|
||||
|
||||
if (match.matches()) {
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
|
||||
} else {
|
||||
// fallback method just in case youtube changes things; it should never run and tests will fail
|
||||
// provides an url with "/user/NAME", that is inconsistent with stream and channel extractor
|
||||
if (match.matches()) {
|
||||
return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
|
||||
}
|
||||
} catch(Exception ignored) {}
|
||||
|
||||
// fallback method for channels without "Subscribe" button (or just in case yt changes things)
|
||||
// provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
|
||||
try {
|
||||
return el.select("a[class*=\"yt-uix-tile-link\"]").first()
|
||||
.attr("abs:href");
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get channel url", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -575,21 +575,26 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
|||
*/
|
||||
@Override
|
||||
public String getErrorMessage() {
|
||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||
StringBuilder errorReason;
|
||||
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
|
||||
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
if (errorElement == null) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
String errorMessage = errorElement.text();
|
||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||
errorReason = null;
|
||||
} else if (errorMessage.contains("GEMA")) {
|
||||
// Gema sometimes blocks youtube music content in germany:
|
||||
// https://www.gema.de/en/
|
||||
// Detailed description:
|
||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||
errorReason = new StringBuilder("GEMA");
|
||||
} else {
|
||||
errorReason = new StringBuilder(errorMessage);
|
||||
errorReason.append(" ");
|
||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||
}
|
||||
}
|
||||
|
||||
return errorReason != null ? errorReason.toString() : null;
|
||||
|
|
|
@ -392,6 +392,100 @@ public class YoutubeChannelExtractorTest {
|
|||
}
|
||||
}
|
||||
|
||||
// this channel has no "Subscribe" button
|
||||
public static class EminemVEVO implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
|
||||
extractor = (YoutubeChannelExtractor) YouTube
|
||||
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// Extractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testServiceId() {
|
||||
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testName() throws Exception {
|
||||
assertEquals("EminemVEVO", extractor.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testId() throws Exception {
|
||||
assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOriginalUrl() throws ParsingException {
|
||||
assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ListExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testRelatedItems() throws Exception {
|
||||
defaultTestRelatedItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMoreRelatedItems() throws Exception {
|
||||
defaultTestMoreItems(extractor, YouTube.getServiceId());
|
||||
}
|
||||
|
||||
/*//////////////////////////////////////////////////////////////////////////
|
||||
// ChannelExtractor
|
||||
//////////////////////////////////////////////////////////////////////////*/
|
||||
|
||||
@Test
|
||||
public void testDescription() throws Exception {
|
||||
final String description = extractor.getDescription();
|
||||
assertTrue(description, description.contains("Eminem on Vevo"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAvatarUrl() throws Exception {
|
||||
String avatarUrl = extractor.getAvatarUrl();
|
||||
assertIsSecureUrl(avatarUrl);
|
||||
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBannerUrl() throws Exception {
|
||||
String bannerUrl = extractor.getBannerUrl();
|
||||
assertIsSecureUrl(bannerUrl);
|
||||
assertTrue(bannerUrl, bannerUrl.contains("yt3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFeedUrl() throws Exception {
|
||||
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
// there is no "Subscribe" button
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertEquals("Wrong subscriber count", -1, subscribers);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static class RandomChannel implements BaseChannelExtractorTest {
|
||||
private static YoutubeChannelExtractor extractor;
|
||||
|
||||
|
@ -483,8 +577,9 @@ public class YoutubeChannelExtractorTest {
|
|||
|
||||
@Test
|
||||
public void testSubscriberCount() throws Exception {
|
||||
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
|
||||
long subscribers = extractor.getSubscriberCount();
|
||||
assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue