From a579337c9a4d317f2c8f247201a57de11c9e196e Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sat, 21 Dec 2019 19:00:07 +0100 Subject: [PATCH] Bandcamp service with support for streams and searches --- extractor/build.gradle | 1 + .../schabi/newpipe/extractor/ServiceList.java | 5 +- .../services/bandcamp/BandcampService.java | 102 ++++++++ .../extractors/BandcampExtractorHelper.java | 83 +++++++ .../extractors/BandcampSearchExtractor.java | 123 ++++++++++ .../extractors/BandcampStreamExtractor.java | 224 ++++++++++++++++++ .../BandcampStreamInfoItemExtractor.java | 87 +++++++ .../BandcampChannelLinkHandlerFactory.java | 59 +++++ .../BandcampSearchQueryHandlerFactory.java | 30 +++ .../BandcampStreamLinkHandlerFactory.java | 47 ++++ ...BandcampChannelLinkHandlerFactoryTest.java | 40 ++++ .../bandcamp/BandcampSearchExtractorTest.java | 55 +++++ ...BandcampSearchQueryHandlerFactoryTest.java | 34 +++ .../bandcamp/BandcampStreamExtractorTest.java | 64 +++++ .../BandcampStreamLinkHandlerFactoryTest.java | 47 ++++ 15 files changed, 1000 insertions(+), 1 deletion(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java diff --git a/extractor/build.gradle b/extractor/build.gradle index 1b7fbf00..cc2f69da 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -6,6 +6,7 @@ dependencies { implementation 'org.mozilla:rhino:1.7.7.1' implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0' implementation 'org.nibor.autolink:autolink:0.8.0' + implementation 'org.json:json:20190722' testImplementation 'junit:junit:4.12' } \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 6be1cea4..742d342d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.schabi.newpipe.extractor.services.bandcamp.BandcampService; import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; @@ -39,6 +40,7 @@ public final class ServiceList { public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; public static final PeertubeService PeerTube; + public static final BandcampService bandcamp; /** * When creating a new service, put this service in the end of this list, @@ -49,7 +51,8 @@ public final class ServiceList { YouTube = new YoutubeService(0), SoundCloud = new SoundcloudService(1), MediaCCC = new MediaCCCService(2), - PeerTube = new PeertubeService(3) + PeerTube = new PeertubeService(3), + bandcamp = new BandcampService(4) )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java new file mode 100644 index 00000000..fd891365 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampService.java @@ -0,0 +1,102 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.util.Collections; + +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + +public class BandcampService extends StreamingService { + + public BandcampService(int id) { + super(id, "bandcamp", Collections.singletonList(AUDIO)); + } + + @Override + public String getBaseUrl() { + return "https://bandcamp.com"; + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return new BandcampStreamLinkHandlerFactory(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + //return new BandcampChannelLinkHandlerFactory(); TODO + return null; + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return new BandcampSearchQueryHandlerFactory(); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { + return new BandcampSearchExtractor(this, queryHandler); + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return null; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return null; + } + + @Override + public KioskList getKioskList() throws ExtractionException { + return null; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException { + return new BandcampStreamExtractor(this, linkHandler); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java new file mode 100644 index 00000000..452c4693 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -0,0 +1,83 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONException; +import org.json.JSONObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.util.ArrayList; +import java.util.Arrays; + +public class BandcampExtractorHelper { + + /** + * Get JSON behind var $variable = out of web page + *
+ * Originally a part of bandcampDirect. + * + * @param html The HTML where the JSON we're looking for is stored inside a + * variable inside some JavaScript block + * @param variable Name of the variable + * @return The JsonObject stored in the variable with this name + */ + public static JSONObject getJSONFromJavaScriptVariables(String html, String variable) throws JSONException, ParsingException { + + String[] part = html.split("var " + variable + " = "); + + String firstHalfGone = part[1]; + + firstHalfGone = firstHalfGone.replaceAll("\" \\+ \"", ""); + + int position = -1; + int level = 0; + for (char character : firstHalfGone.toCharArray()) { + position++; + + switch (character) { + case '{': + level++; + continue; + case '}': + level--; + if (level == 0) { + return new JSONObject(firstHalfGone.substring(0, position + 1) + .replaceAll(" {4}//.+", "") // Remove comments in JSON + ); + } + } + } + + throw new ParsingException("Unexpected HTML: JSON never ends"); + } + + /** + * Concatenate all non-null and non-empty strings together while separating them using + * the comma parameter + */ + public static String smartConcatenate(String[] strings, String comma) { + StringBuilder result = new StringBuilder(); + + // Remove empty strings + ArrayList list = new ArrayList<>(Arrays.asList(strings)); + for (int i = list.size() - 1; i >= 0; i--) { + if (list.get(i) == null || list.get(i).isEmpty()) { + list.remove(i); + } + } + + // Append remaining strings to result + for (int i = 0; i < list.size(); i++) { + String string = list.get(i); + result.append(string); + + if (i != list.size() - 1) { + // This is not the last iteration yet + result.append(comma); + } + + } + + return String.valueOf(result); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java new file mode 100644 index 00000000..1b49ff70 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java @@ -0,0 +1,123 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class BandcampSearchExtractor extends SearchExtractor { + + public BandcampSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public String getSearchSuggestion() { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + // okay apparently this is where we DOWNLOAD the page and then COMMIT its ENTRIES to an INFOITEMPAGE + String html = getDownloader().get(pageUrl).responseBody(); + + InfoItemsSearchCollector collector = getInfoItemSearchCollector(); + + + Document d = Jsoup.parse(html); + + Elements searchResultsElements = d.getElementsByClass("searchresult"); + + for (Element searchResult : + searchResultsElements) { + + Element resultInfo = searchResult.getElementsByClass("result-info").first(); + + String type = resultInfo + .getElementsByClass("itemtype").first().text(); + + String image = null; + Element img = searchResult.getElementsByClass("art").first() + .getElementsByTag("img").first(); + if (img != null) { + image = img.attr("src"); + } + + String heading = resultInfo.getElementsByClass("heading").text(); + + String subhead = resultInfo.getElementsByClass("subhead").text(); + + String url = resultInfo.getElementsByClass("itemurl").text(); + + switch (type) { + default: + continue; + case "FAN": + //collector.commit Channel (?) with heading, url, image + break; + + case "ARTIST": + String id = resultInfo.getElementsByClass("itemurl").first() + .getElementsByTag("a").first() + .attr("href") // the link contains the id + .split("search_item_id=") + [1] // the number is behind its name + .split("&") // there is another attribute behind the name + [0]; // get the number + + //searchResults.add(new Artist(heading, Long.parseLong(id), image, subhead)); + //collector.commit Channel with heading, id, image, subhead + break; + + case "ALBUM": + String artist = subhead.split(" by")[0]; + //searchResults.add(new Album(heading, artist, url, image)); + //collector.commit Playlist with heading, artist, url, image + break; + + case "TRACK": + String album = subhead.split("from ")[0].split(" by")[0]; + + String[] splitBy = subhead.split(" by"); + String artist1 = null; + if (splitBy.length > 1) { + artist1 = subhead.split(" by")[1]; + } + collector.commit(new BandcampStreamInfoItemExtractor(heading, url, image, artist1, album)); + break; + } + + } + + + return new InfoItemsPage<>(getInfoItemSearchCollector(), null); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getUrl());//new InfoItemsPage<>(getInfoItemSearchCollector(), null); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java new file mode 100644 index 00000000..0bb100c0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java @@ -0,0 +1,224 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.json.JSONException; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.*; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.List; + +public class BandcampStreamExtractor extends StreamExtractor { + + private JSONObject albumJson; + private JSONObject current; + private Document document; + + public BandcampStreamExtractor(StreamingService service, LinkHandler linkHandler) { + super(service, linkHandler); + } + + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String html = downloader.get(getLinkHandler().getUrl()).responseBody(); + document = Jsoup.parse(html); + albumJson = getAlbumInfoJson(html); + current = albumJson.getJSONObject("current"); + + if (albumJson.getJSONArray("trackinfo").length() > 1) { + // In this case, we are actually viewing an album page! + throw new ExtractionException("Page is actually an album, not a track"); + } + } + + /** + * Get the JSON that contains album's metadata from page + * + * @param html Website + * @return Album metadata JSON + * @throws ParsingException In case of a faulty website + */ + public static JSONObject getAlbumInfoJson(String html) throws ParsingException { + try { + return BandcampExtractorHelper.getJSONFromJavaScriptVariables(html, "TralbumData"); + } catch (JSONException e) { + throw new ParsingException("Faulty JSON", e); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return current.getString("title"); + } + + @Nonnull + @Override + public String getUploaderUrl() throws ParsingException { + String[] parts = getUrl().split("/"); + // https: (/) (/) * .bandcamp.com (/) and leave out the rest + return "https://" + parts[2] + "/"; + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return albumJson.getString("url").replace("http://", "https://"); + } + + @Nonnull + @Override + public String getUploaderName() throws ParsingException { + return albumJson.getString("artist"); + } + + @Nullable + @Override + public String getTextualUploadDate() throws ParsingException { + return current.getString("release_date"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return null; + } + + @Nonnull + @Override + public String getThumbnailUrl() throws ParsingException { + return document.getElementsByAttributeValue("property", "og:image").get(0).attr("content"); + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() { + return document.getElementsByClass("band-photo").first().attr("src"); + } + + @Nonnull + @Override + public String getDescription() { + return BandcampExtractorHelper.smartConcatenate( + new String[]{ + getStringOrNull(current, "about"), + getStringOrNull(current, "lyrics"), + getStringOrNull(current, "credits") + }, "\n\n" + ); + } + + /** + * Avoid exceptions like "JSONObject["about"] not a string." and instead just return null. + * This is for the case that the actual JSON has something like "about": null. + */ + private String getStringOrNull(JSONObject jsonObject, String value) { + try { + return jsonObject.getString(value); + } catch (JSONException e) { + return null; + } + } + + @Override + public int getAgeLimit() throws ParsingException { + return 0; + } + + @Override + public long getLength() throws ParsingException { + return 0; + } + + @Override + public long getTimeStamp() throws ParsingException { + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + return -1; + } + + @Override + public long getLikeCount() throws ParsingException { + return -1; + } + + @Override + public long getDislikeCount() throws ParsingException { + return -1; + } + + @Nonnull + @Override + public String getDashMpdUrl() throws ParsingException { + return null; + } + + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + return null; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + return null; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return null; + } + + @Override + public StreamInfoItem getNextStream() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public String getErrorMessage() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java new file mode 100644 index 00000000..d1d96579 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamInfoItemExtractor.java @@ -0,0 +1,87 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.extractors; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; + +public class BandcampStreamInfoItemExtractor implements StreamInfoItemExtractor { + + private String title; + private String url; + private String cover; + private String artist; + private String albumName; + + public BandcampStreamInfoItemExtractor(String title, String url, String cover, String artist, String albumName) { + this.title = title; + this.url = url; + this.cover = cover; + this.artist = artist; + this.albumName = albumName; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.AUDIO_STREAM; + } + + @Override + public long getDuration() throws ParsingException { + return -1; + } + + @Override + public long getViewCount() throws ParsingException { + return -1; + } + + @Override + public String getUploaderName() throws ParsingException { + return artist; + } + + @Override + public String getUploaderUrl() throws ParsingException { + return null; + } + + @Nullable + @Override + public String getTextualUploadDate() throws ParsingException { + return null; // TODO + } + + @Nullable + @Override + public DateWrapper getUploadDate() throws ParsingException { + return null; + } + + @Override + public String getName() throws ParsingException { + return title; + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return cover; + } + + /** + * There are no ads just like that, duh + */ + @Override + public boolean isAd() throws ParsingException { + return false; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java new file mode 100644 index 00000000..24f19f76 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java @@ -0,0 +1,59 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; + +import java.io.IOException; +import java.util.List; + +/** + * Artist do have IDs that are useful + */ +public class BandcampChannelLinkHandlerFactory extends ListLinkHandlerFactory { + + + @Override + public String getId(String url) throws ParsingException { + try { + String response = NewPipe.getDownloader().get(url).responseBody(); + return BandcampStreamExtractor.getAlbumInfoJson(response) + .getString("band_id"); + } catch (IOException | ReCaptchaException e) { + throw new ParsingException("Download failed", e); + } + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return null; // TODO + } + + /** + * Matches * .bandcamp.com as well as custom domains + * where the profile is at * . * /releases + */ + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + + // Ends with "bandcamp.com" or "bandcamp.com/"? + boolean endsWithBandcampCom = url.endsWith("bandcamp.com") + || url.endsWith("bandcamp.com/"); + + // Is a subdomain of bandcamp.com? + boolean isBandcampComSubdomain = url.matches("https?://.+\\.bandcamp\\.com"); + + // Is root of bandcamp.com subdomain? + boolean isBandcampComArtistPage = endsWithBandcampCom && isBandcampComSubdomain; + + boolean isCustomDomainReleases = url.matches("https?://.+\\..+/releases/?(?!.)"); + + return isBandcampComArtistPage || isCustomDomainReleases; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java new file mode 100644 index 00000000..3f23956f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java @@ -0,0 +1,30 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + private static final String SEARCH_URL = "https://bandcamp.com/search?q="; + + public static final String CHARSET_UTF_8 = "UTF-8"; + + + @Override + public String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException { + try { + + return SEARCH_URL + + URLEncoder.encode(query, CHARSET_UTF_8); + + } catch (UnsupportedEncodingException e) { + throw new ParsingException("query \"" + query + "\" could not be encoded", e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java new file mode 100644 index 00000000..e534f925 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java @@ -0,0 +1,47 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; + +/** + * Tracks do have IDs, but they are not really useful. That's why id = url. + * Instead, URLs are cleaned up so that they always look the same. + */ +public class BandcampStreamLinkHandlerFactory extends LinkHandlerFactory { + + + /** + * @see BandcampStreamLinkHandlerFactory + */ + @Override + public String getId(String url) throws ParsingException { + return getUrl(url); + } + + /** + * Clean up url + * @see BandcampStreamLinkHandlerFactory + */ + @Override + public String getUrl(String url) { + if (url.endsWith("/")) + url = url.substring(0, url.length() - 1); + url = url.replace("http://", "https://").toLowerCase(); + return url; + } + + /** + * Sometimes, the root page of an artist is also an album or track + * page. In that case, it is assumed that one actually wants to open + * the profile and not the track it has set as the default one. + *

Urls are expected to be in this format to account for + * custom domains: + *
https:// * . * /track/ * + */ + @Override + public boolean onAcceptUrl(String url) { + return getUrl(url).matches("https?://.+\\..+/track/.+"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java new file mode 100644 index 00000000..d8c7497d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampChannelLinkHandlerFactoryTest.java @@ -0,0 +1,40 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampChannelLinkHandlerFactory; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; + +/** + * Test for {@link BandcampChannelLinkHandlerFactory} + */ +public class BandcampChannelLinkHandlerFactoryTest { + private static BandcampChannelLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampChannelLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting true + assertTrue(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertTrue(linkHandler.acceptUrl("https://interovgm.com/releases")); + assertTrue(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + + // Tests expecting false + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java new file mode 100644 index 00000000..11ce3d0d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchExtractorTest.java @@ -0,0 +1,55 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampSearchExtractor; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +/** + * Test for {@link BandcampSearchExtractor} + */ +public class BandcampSearchExtractorTest { + + private static BandcampSearchExtractor extractor; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + + } + + /** + * Tests whether searching bandcamp for "best friend's basement" returns + * the accordingly named song by Zach Benson + */ + @Test + public void testBestFriendsBasement() throws ExtractionException, IOException { + extractor = (BandcampSearchExtractor) bandcamp + .getSearchExtractor("best friend's basement"); + + ListExtractor.InfoItemsPage page = extractor.getInitialPage(); + InfoItem bestFriendsBasement = page.getItems().get(0); + + // The track by Zach Benson should be the first result, no? + assertEquals("Best Friend's Basement", bestFriendsBasement.getName()); + assertTrue(bestFriendsBasement.getThumbnailUrl().endsWith(".jpg")); + assertTrue(bestFriendsBasement.getThumbnailUrl().contains("f4.bcbits.com/img/")); + assertEquals(InfoItem.InfoType.STREAM, bestFriendsBasement.getInfoType()); + + + + + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java new file mode 100644 index 00000000..1a8fdac5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampSearchQueryHandlerFactoryTest.java @@ -0,0 +1,34 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampSearchQueryHandlerFactory; + +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampSearchQueryHandlerFactoryTest { + + static BandcampSearchQueryHandlerFactory searchQuery; + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + + searchQuery = (BandcampSearchQueryHandlerFactory) bandcamp + .getSearchQHFactory(); + } + + @Test + public void testEncoding() throws ParsingException { + // Note: this isn't exactly as bandcamp does it (it wouldn't encode '!'), but both works + assertEquals("https://bandcamp.com/search?q=hello%21%22%C2%A7%24%25%26%2F%28%29%3D", searchQuery.getUrl("hello!\"ยง$%&/()=")); + // Note: bandcamp uses %20 instead of '+', but both works + assertEquals("https://bandcamp.com/search?q=search+query+with+spaces", searchQuery.getUrl("search query with spaces")); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java new file mode 100644 index 00000000..f64e5b55 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamExtractorTest.java @@ -0,0 +1,64 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.bandcamp; + +public class BandcampStreamExtractorTest { + + private static BandcampStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = (BandcampStreamExtractor) bandcamp + .getStreamExtractor("https://zachbenson.bandcamp.com/track/kitchen"); + extractor.fetchPage(); + } + + @Test(expected = ExtractionException.class) + public void testAlbum() throws ExtractionException { + bandcamp.getStreamExtractor("https://zachbenson.bandcamp.com/album/prom"); + } + + @Test + public void testServiceId() { + } + + @Test + public void testName() throws ParsingException { + assertEquals("kitchen", extractor.getName()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/track/kitchen", extractor.getUrl()); + } + + @Test + public void testArtistUrl() throws ParsingException { + assertEquals("https://zachbenson.bandcamp.com/", extractor.getUploaderUrl()); + } + + @Test + public void testDescription() { + assertEquals(831, extractor.getDescription().length()); + } + + @Test + public void testArtistProfilePicture() { + String url = extractor.getUploaderAvatarUrl(); + assertTrue(url.contains("://f4.bcbits.com/img/") && url.endsWith(".jpg")); + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java new file mode 100644 index 00000000..0eb76077 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampStreamLinkHandlerFactoryTest.java @@ -0,0 +1,47 @@ +// Created by Fynn Godau 2019, licensed GNU GPL version 3 or later + +package org.schabi.newpipe.extractor.services.bandcamp; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.bandcamp.linkHandler.BandcampStreamLinkHandlerFactory; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; + +/** + * Test for {@link BandcampStreamLinkHandlerFactory} + */ +public class BandcampStreamLinkHandlerFactoryTest { + + private static BandcampStreamLinkHandlerFactory linkHandler; + + @BeforeClass + public static void setUp() { + linkHandler = new BandcampStreamLinkHandlerFactory(); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testUrlCleanup() { + assertEquals("https://zachbenson.bandcamp.com/track/u-i-tonite", linkHandler.getUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); + } + + @Test + public void testAcceptUrl() throws ParsingException { + // Tests expecting false + assertFalse(linkHandler.acceptUrl("http://interovgm.com/releases/")); + assertFalse(linkHandler.acceptUrl("https://interovgm.com/releases")); + assertFalse(linkHandler.acceptUrl("http://zachbenson.bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://bandcamp.com")); + assertFalse(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/")); + + // Tests expecting true + assertTrue(linkHandler.acceptUrl("https://zachbenson.bandcamp.com/track/kitchen")); + assertTrue(linkHandler.acceptUrl("http://ZachBenson.Bandcamp.COM/Track/U-I-Tonite/")); + assertTrue(linkHandler.acceptUrl("https://interovgm.com/track/title")); + } +}