From 466865385e565cb1777d631efa8bfd262467a04b Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 29 Apr 2018 21:31:24 +0200 Subject: [PATCH] add support for affiliate --- .../soundcloud/SoundcloudStreamExtractor.java | 5 ++++ .../extractor/services/youtube/ItagItem.java | 3 ++ .../youtube/YoutubeChannelExtractor.java | 2 +- .../youtube/YoutubeStreamExtractor.java | 19 +++++++++++- .../extractor/stream/StreamExtractor.java | 1 + .../newpipe/extractor/stream/StreamInfo.java | 29 +++++++++++++++++++ .../extractor/utils/DonationLinkHelper.java | 16 ++++++++-- .../YoutubeStreamExtractorDonationTest.java | 6 ++++ 8 files changed, 77 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 8de597d8..30478b59 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -214,6 +214,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor { return new String[0]; } + @Override + public String[] getAffiliateLinks() { + return new String[0]; + } + @Override public String getErrorMessage() { return null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 1950c0b5..3be29c98 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -43,6 +43,9 @@ public class ItagItem { new ItagItem(139, AUDIO, M4A, 48), new ItagItem(140, AUDIO, M4A, 128), new ItagItem(141, AUDIO, M4A, 256), + new ItagItem(249, AUDIO, OPUS, 50), + new ItagItem(250, AUDIO, OPUS, 70), + new ItagItem(160, AUDIO, OPUS, 160), /// VIDEO ONLY //////////////////////////////////////////// // ID Type Format Resolution FPS /// diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java index 195d20ab..dfcfd6b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java @@ -196,7 +196,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } for(Element a : linkHolder.select("a")) { String link = a.attr("abs:href"); - if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { + if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { links.add(link); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index c9839ac6..a5d48a14 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -534,7 +534,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { ArrayList donationLinks = new ArrayList<>(); for (String s : Parser.getLinksFromString(getDescription())) { - if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { + if (DonationLinkHelper.getDonatoinServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) { donationLinks.add(s); } } @@ -546,6 +546,23 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + @Override + public String[] getAffiliateLinks() throws ParsingException { + try { + ArrayList donationLinks = new ArrayList<>(); + for (String s : Parser.getLinksFromString(getDescription())) { + if (DonationLinkHelper.getAffiliateServiceByLink(s) != DonationLinkHelper.AffiliateService.NO_AFILIATE) { + donationLinks.add(s); + } + } + String[] donlret = new String[donationLinks.size()]; + donlret = donationLinks.toArray(donlret); + return donlret; + } catch (Exception e) { + throw new ParsingException("Could not get afiliate links", e); + } + } + /*////////////////////////////////////////////////////////////////////////// // Fetch page diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 0aeedd02..8a2626b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -142,6 +142,7 @@ public abstract class StreamExtractor extends Extractor { public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException; public abstract String[] getDonationLinks() throws ExtractionException; + public abstract String[] getAffiliateLinks() throws ExtractionException; /** * Analyses the webpage's document and extracts any error message there might be. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 716afa4b..50b2ec0f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -242,6 +242,16 @@ public class StreamInfo extends Info { } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setAffiliateLinks(extractor.getAffiliateLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setDonationLinks(extractor.getDonationLinks()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -274,6 +284,9 @@ public class StreamInfo extends Info { private long startPosition = 0; private List subtitles; + private String[] donationLinks; + private String[] affiliateLinks; + /** * Get the stream type * @@ -466,4 +479,20 @@ public class StreamInfo extends Info { public void setSubtitles(List subtitles) { this.subtitles = subtitles; } + + public String[] getDonationLinks() { + return donationLinks; + } + + public void setDonationLinks(String[] donationLinks) { + this.donationLinks = donationLinks; + } + + public String[] getAffiliateLinks() { + return affiliateLinks; + } + + public void setAffiliateLinks(String[] affiliateLinks) { + this.affiliateLinks = affiliateLinks; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java index b62e062f..04913225 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java @@ -7,11 +7,16 @@ public class DonationLinkHelper { public enum DonationService { NO_DONATION, PATREON, - PAYPAL + PAYPAL, + } + + public enum AffiliateService { + NO_AFILIATE, + AMAZON, } - public static DonationService getServiceByLink(String link) throws MalformedURLException { + public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException { URL url = new URL(link); switch (url.getHost()) { case "www.patreon.com": @@ -27,5 +32,12 @@ public class DonationLinkHelper { } } + public static AffiliateService getAffiliateServiceByLink(String link) throws MalformedURLException { + URL url = new URL(link); + switch (url.getHost()) { + case "amzn.to": return AffiliateService.AMAZON; + default: return AffiliateService.NO_AFILIATE; + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java index 8bc82ded..9f26a7bd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java @@ -49,4 +49,10 @@ public class YoutubeStreamExtractorDonationTest { assertTrue(String.valueOf(extractor.getDonationLinks().length), extractor.getDonationLinks().length == 2); } + + @Test + public void getAffiliateLinksTest() throws Exception { + assertTrue(String.valueOf(extractor.getAffiliateLinks().length), + extractor.getAffiliateLinks().length == 1); + } }