CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #811 from TiA4f8R/playlists-improvements-and-yt-playlists-fixes 

[YouTube] Fix the extraction of series playlists and don't return the view count as the stream count for learning playlists
This commit is contained in:
litetex 2022-03-17 13:52:24 +01:00 committed by GitHub
parent 40aa5104b1 c7757c0994
commit cd8088b217
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 226 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java
View File

@ -8,15 +8,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> { public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public PlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { public PlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getBannerUrl() throws ParsingException;
public abstract String getUploaderUrl() throws ParsingException; public abstract String getUploaderUrl() throws ParsingException;
public abstract String getUploaderName() throws ParsingException; public abstract String getUploaderName() throws ParsingException;
public abstract String getUploaderAvatarUrl() throws ParsingException; public abstract String getUploaderAvatarUrl() throws ParsingException;
@ -24,8 +23,30 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public abstract long getStreamCount() throws ParsingException; public abstract long getStreamCount() throws ParsingException;
@Nonnull public abstract String getSubChannelName() throws ParsingException; @Nonnull
@Nonnull public abstract String getSubChannelUrl() throws ParsingException; public String getThumbnailUrl() throws ParsingException {
@Nonnull public abstract String getSubChannelAvatarUrl() throws ParsingException; return EMPTY_STRING;
}
@Nonnull
public String getBannerUrl() throws ParsingException {
// Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement it in the frontend.
return EMPTY_STRING;
}
@Nonnull
public String getSubChannelName() throws ParsingException {
return EMPTY_STRING;
}
@Nonnull
public String getSubChannelUrl() throws ParsingException {
return EMPTY_STRING;
}
@Nonnull
public String getSubChannelAvatarUrl() throws ParsingException {
return EMPTY_STRING;
}
} }

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistExtractor.java
View File

@ -19,10 +19,13 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData; import static org.schabi.newpipe.extractor.utils.JsonUtils.getJsonData;
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson; import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampStreamExtractor.getAlbumInfoJson;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
public class BandcampPlaylistExtractor extends PlaylistExtractor { public class BandcampPlaylistExtractor extends PlaylistExtractor {
@ -57,33 +60,27 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
throw new ParsingException("JSON does not exist", e); throw new ParsingException("JSON does not exist", e);
} }
if (trackInfo.isEmpty()) {
if (trackInfo.size() <= 0) {
// Albums without trackInfo need to be purchased before they can be played // Albums without trackInfo need to be purchased before they can be played
throw new ContentNotAvailableException("Album needs to be purchased"); throw new ContentNotAvailableException("Album needs to be purchased");
} }
} }
@Nonnull
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
if (albumJson.isNull("art_id")) { if (albumJson.isNull("art_id")) {
return ""; return EMPTY_STRING;
} else { } else {
return getImageUrl(albumJson.getLong("art_id"), true); return getImageUrl(albumJson.getLong("art_id"), true);
} }
} }
@Override
public String getBannerUrl() {
return "";
}
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
final String[] parts = getUrl().split("/"); final String[] parts = getUrl().split("/");
// https: (/) (/) * .bandcamp.com (/) and leave out the rest // https: (/) (/) * .bandcamp.com (/) and leave out the rest
return "https://" + parts[2] + "/"; return HTTPS + parts[2] + "/";
} }
@Override @Override
@ -94,9 +91,10 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
try { try {
return document.getElementsByClass("band-photo").first().attr("src"); return Objects.requireNonNull(document.getElementsByClass("band-photo").first())
} catch (NullPointerException e) { .attr("src");
return ""; } catch (final NullPointerException e) {
return EMPTY_STRING;
} }
} }
@ -110,24 +108,6 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
return trackInfo.size(); return trackInfo.size();
} }
@Nonnull
@Override
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
@ -146,14 +126,13 @@ public class BandcampPlaylistExtractor extends PlaylistExtractor {
collector.commit(new BandcampPlaylistStreamInfoItemExtractor( collector.commit(new BandcampPlaylistStreamInfoItemExtractor(
track, getUploaderUrl(), getThumbnailUrl())); track, getUploaderUrl(), getThumbnailUrl()));
} }
} }
return new InfoItemsPage<>(collector, null); return new InfoItemsPage<>(collector, null);
} }
@Override @Override
public InfoItemsPage<StreamInfoItem> getPage(Page page) { public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return null; return null;
} }

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -29,16 +29,12 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
super(service, linkHandler); super(service, linkHandler);
} }
@Nonnull
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
return getBaseUrl() + playlistInfo.getString("thumbnailPath"); return getBaseUrl() + playlistInfo.getString("thumbnailPath");
} }
@Override
public String getBannerUrl() {
return null;
}
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
return playlistInfo.getObject("ownerAccount").getString("url"); return playlistInfo.getObject("ownerAccount").getString("url");

61
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View File

@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudPlaylistExtractor extends PlaylistExtractor { public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@ -67,7 +67,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return playlist.getString("title"); return playlist.getString("title");
} }
@Nullable @Nonnull
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() {
String artworkUrl = playlist.getString("artwork_url"); String artworkUrl = playlist.getString("artwork_url");
@ -80,24 +80,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
for (final StreamInfoItem item : infoItems.getItems()) { for (final StreamInfoItem item : infoItems.getItems()) {
artworkUrl = item.getThumbnailUrl(); artworkUrl = item.getThumbnailUrl();
if (!isNullOrEmpty(artworkUrl)) break; if (!isNullOrEmpty(artworkUrl)) {
break;
}
} }
} catch (final Exception ignored) { } catch (final Exception ignored) {
} }
if (artworkUrl == null) { if (artworkUrl == null) {
return null; return EMPTY_STRING;
} }
} }
return artworkUrl.replace("large.jpg", "crop.jpg"); return artworkUrl.replace("large.jpg", "crop.jpg");
} }
@Override
public String getBannerUrl() {
return null;
}
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
return SoundcloudParsingHelper.getUploaderUrl(playlist); return SoundcloudParsingHelper.getUploaderUrl(playlist);
@ -123,24 +120,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return playlist.getLong("track_count"); return playlist.getLong("track_count");
} }
@Nonnull
@Override
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() { public InfoItemsPage<StreamInfoItem> getInitialPage() {
@ -148,19 +127,21 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
new StreamInfoItemsCollector(getServiceId()); new StreamInfoItemsCollector(getServiceId());
final List<String> ids = new ArrayList<>(); final List<String> ids = new ArrayList<>();
final JsonArray tracks = playlist.getArray("tracks"); playlist.getArray("tracks")
for (final Object o : tracks) { .stream()
if (o instanceof JsonObject) { .filter(JsonObject.class::isInstance)
final JsonObject track = (JsonObject) o; .map(JsonObject.class::cast)
if (track.has("title")) { // i.e. if full info is available .forEachOrdered(track -> {
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track)); // i.e. if full info is available
} else { if (track.has("title")) {
// %09d would be enough, but a 0 before the number does not create problems, so streamInfoItemsCollector.commit(
// let's be sure new SoundcloudStreamInfoItemExtractor(track));
ids.add(String.format("%010d", track.getInt("id"))); } else {
} // %09d would be enough, but a 0 before the number does not create
} // problems, so let's be sure
} ids.add(String.format("%010d", track.getInt("id")));
}
});
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(ids)); return new InfoItemsPage<>(streamInfoItemsCollector, new Page(ids));
} }

67
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.*;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -71,7 +72,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString)); jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString));
} }
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(StandardCharsets.UTF_8);
final Map<String, List<String>> headers = new HashMap<>(); final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers); addClientInfoHeaders(headers);
@ -97,6 +98,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return name; return name;
} }
@Nonnull
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
@ -108,19 +110,15 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
.getObject("watchEndpoint").getString("videoId")); .getObject("watchEndpoint").getString("videoId"));
} catch (final Exception ignored) { } catch (final Exception ignored) {
} }
throw new ParsingException("Could not get playlist thumbnail", e); throw new ParsingException("Could not get playlist thumbnail", e);
} }
} }
@Override
public String getBannerUrl() {
return "";
}
@Override @Override
public String getUploaderUrl() { public String getUploaderUrl() {
// YouTube mixes are auto-generated by YouTube // YouTube mixes are auto-generated by YouTube
return ""; return EMPTY_STRING;
} }
@Override @Override
@ -132,7 +130,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
// YouTube mixes are auto-generated by YouTube // YouTube mixes are auto-generated by YouTube
return ""; return EMPTY_STRING;
} }
@Override @Override
@ -148,8 +146,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, public InfoItemsPage<StreamInfoItem> getInitialPage()
ExtractionException { throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, playlistData.getArray("contents")); collectStreamsFrom(collector, playlistData.getArray("contents"));
@ -159,9 +157,10 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies)); return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies));
} }
private Page getNextPageFrom(final JsonObject playlistJson, @Nonnull
final Map<String, String> cookies) throws IOException, private Page getNextPageFrom(@Nonnull final JsonObject playlistJson,
ExtractionException { final Map<String, String> cookies)
throws IOException, ExtractionException {
final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents")
.get(playlistJson.getArray("contents").size() - 1)); .get(playlistJson.getArray("contents").size() - 1));
if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) {
@ -181,7 +180,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
.value("playlistIndex", index) .value("playlistIndex", index)
.value("params", params) .value("params", params)
.done()) .done())
.getBytes(UTF_8); .getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body); return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
} }
@ -217,26 +216,23 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nullable final List<Object> streams) { @Nullable final List<Object> streams) {
if (streams == null) { if (streams == null) {
return; return;
} }
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object stream : streams) { streams.stream()
if (stream instanceof JsonObject) { .filter(JsonObject.class::isInstance)
final JsonObject streamInfo = ((JsonObject) stream) .map(JsonObject.class::cast)
.getObject("playlistPanelVideoRenderer"); .map(stream -> stream.getObject("playlistPanelVideoRenderer"))
if (streamInfo != null) { .filter(Objects::nonNull)
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, .map(streamInfo -> new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser))
timeAgoParser)); .forEachOrdered(collector::commit);
}
}
}
} }
private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { @Nonnull
private String getThumbnailUrlFromPlaylistId(@Nonnull final String playlistId) throws ParsingException {
final String videoId; final String videoId;
if (playlistId.startsWith("RDMM")) { if (playlistId.startsWith("RDMM")) {
videoId = playlistId.substring(4); videoId = playlistId.substring(4);
@ -251,25 +247,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return getThumbnailUrlFromVideoId(videoId); return getThumbnailUrlFromVideoId(videoId);
} }
@Nonnull
private String getThumbnailUrlFromVideoId(final String videoId) { private String getThumbnailUrlFromVideoId(final String videoId) {
return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg";
} }
@Nonnull
@Override
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() {
return "";
}
} }

200
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -21,22 +21,31 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject initialData; // Minimum size of the stats array in the browse response which includes the streams count
private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2;
// Names of some objects in JSON response frequently used in this class
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
private JsonObject browseResponse;
private JsonObject playlistInfo; private JsonObject playlistInfo;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { public YoutubePlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler); super(service, linkHandler);
} }
@ -45,41 +54,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
ExtractionException { ExtractionException {
final Localization localization = getExtractorLocalization(); final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry()) getExtractorContentCountry())
.value("browseId", "VL" + getId()) .value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // Show unavailable videos .value("params", "wgYCCAA%3D") // Show unavailable videos
.done()) .done())
.getBytes(UTF_8); .getBytes(StandardCharsets.UTF_8);
initialData = getJsonPostResponse("browse", body, localization); browseResponse = getJsonPostResponse("browse", body, localization);
YoutubeParsingHelper.defaultAlertsCheck(initialData); YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
playlistInfo = getPlaylistInfo(); playlistInfo = getPlaylistInfo();
} }
private JsonObject getUploaderInfo() throws ParsingException { private JsonObject getUploaderInfo() throws ParsingException {
final JsonArray items = initialData.getObject("sidebar") final JsonArray items = browseResponse.getObject("sidebar")
.getObject("playlistSidebarRenderer").getArray("items"); .getObject("playlistSidebarRenderer")
.getArray("items");
JsonObject videoOwner = items.getObject(1) JsonObject videoOwner = items.getObject(1)
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); .getObject("playlistSidebarSecondaryInfoRenderer")
if (videoOwner.has("videoOwnerRenderer")) { .getObject("videoOwner");
return videoOwner.getObject("videoOwnerRenderer"); if (videoOwner.has(VIDEO_OWNER_RENDERER)) {
return videoOwner.getObject(VIDEO_OWNER_RENDERER);
} }
// we might want to create a loop here instead of using duplicated code // we might want to create a loop here instead of using duplicated code
videoOwner = items.getObject(items.size()) videoOwner = items.getObject(items.size())
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner"); .getObject("playlistSidebarSecondaryInfoRenderer")
if (videoOwner.has("videoOwnerRenderer")) { .getObject("videoOwner");
return videoOwner.getObject("videoOwnerRenderer"); if (videoOwner.has(VIDEO_OWNER_RENDERER)) {
return videoOwner.getObject(VIDEO_OWNER_RENDERER);
} }
throw new ParsingException("Could not get uploader info"); throw new ParsingException("Could not get uploader info");
} }
private JsonObject getPlaylistInfo() throws ParsingException { private JsonObject getPlaylistInfo() throws ParsingException {
try { try {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer") return browseResponse.getObject("sidebar")
.getArray("items").getObject(0) .getObject("playlistSidebarRenderer")
.getArray("items")
.getObject(0)
.getObject("playlistSidebarPrimaryInfoRenderer"); .getObject("playlistSidebarPrimaryInfoRenderer");
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get PlaylistInfo", e); throw new ParsingException("Could not get PlaylistInfo", e);
@ -90,33 +104,41 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title")); final String name = getTextFromObject(playlistInfo.getObject("title"));
if (!isNullOrEmpty(name)) return name; if (!isNullOrEmpty(name)) {
return name;
}
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title"); return browseResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getString("title");
} }
@Nonnull
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer") String url = playlistInfo.getObject("thumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); .getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
if (isNullOrEmpty(url)) { if (isNullOrEmpty(url)) {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail") url = browseResponse.getObject("microformat")
.getArray("thumbnails").getObject(0).getString("url"); .getObject("microformatDataRenderer")
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
if (isNullOrEmpty(url)) throw new ParsingException("Could not get playlist thumbnail"); if (isNullOrEmpty(url)) {
throw new ParsingException("Could not get playlist thumbnail");
}
} }
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} }
@Override
public String getBannerUrl() {
// Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement it in the frontend.
return "";
}
@Override @Override
public String getUploaderUrl() throws ParsingException { public String getUploaderUrl() throws ParsingException {
try { try {
@ -138,7 +160,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public String getUploaderAvatarUrl() throws ParsingException { public String getUploaderAvatarUrl() throws ParsingException {
try { try {
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url"); final String url = getUploaderInfo()
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
return fixThumbnailUrl(url); return fixThumbnailUrl(url);
} catch (final Exception e) { } catch (final Exception e) {
@ -148,14 +174,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public boolean isUploaderVerified() throws ParsingException { public boolean isUploaderVerified() throws ParsingException {
// YouTube doesn't provide this information
return false; return false;
} }
@Override @Override
public long getStreamCount() throws ParsingException { public long getStreamCount() throws ParsingException {
try { try {
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0)); final JsonArray stats = playlistInfo.getArray("stats");
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText)); // For unknown reasons, YouTube don't provide the stream count for learning playlists
// on the desktop client but only the number of views and the playlist modified date
// On normal playlists, at least 3 items are returned: the number of videos, the number
// of views and the playlist modification date
// We can get it by using another client, however it seems we can't get the avatar
// uploader URL with another client than the WEB client
if (stats.size() > STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE) {
final String videosText = getTextFromObject(playlistInfo.getArray("stats")
.getObject(0));
if (videosText != null) {
return Long.parseLong(Utils.removeNonDigitCharacters(videosText));
}
}
return ITEM_COUNT_UNKNOWN;
} catch (final Exception e) { } catch (final Exception e) {
throw new ParsingException("Could not get video count from playlist", e); throw new ParsingException("Could not get video count from playlist", e);
} }
@ -164,19 +205,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull @Nonnull
@Override @Override
public String getSubChannelName() { public String getSubChannelName() {
return ""; return EMPTY_STRING;
} }
@Nonnull @Nonnull
@Override @Override
public String getSubChannelUrl() { public String getSubChannelUrl() {
return ""; return EMPTY_STRING;
} }
@Nonnull @Nonnull
@Override @Override
public String getSubChannelAvatarUrl() { public String getSubChannelAvatarUrl() {
return ""; return EMPTY_STRING;
} }
@Nonnull @Nonnull
@ -185,26 +226,32 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null; Page nextPage = null;
final JsonArray contents = initialData.getObject("contents") final JsonArray contents = browseResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0) .getObject("twoColumnBrowseResultsRenderer")
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer") .getArray("tabs")
.getArray("contents").getObject(0).getObject("itemSectionRenderer") .getObject(0)
.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents"); .getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) { final JsonObject videoPlaylistObject = contents.stream()
for (final Object segment : contents) { .filter(JsonObject.class::isInstance)
if (((JsonObject) segment).getObject("playlistSegmentRenderer") .map(JsonObject.class::cast)
.has("videoList")) { .map(content -> content.getObject("itemSectionRenderer")
collectStreamsFrom(collector, ((JsonObject) segment) .getArray("contents")
.getObject("playlistSegmentRenderer").getObject("videoList") .getObject(0))
.getObject("playlistVideoListRenderer").getArray("contents")); .filter(contentItemSectionRendererContents ->
} contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
} || contentItemSectionRendererContents.has(
"playlistSegmentRenderer"))
.findFirst()
.orElse(null);
return new InfoItemsPage<>(collector, null); if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
} else if (contents.getObject(0).has("playlistVideoListRenderer")) { final JsonArray videosArray = videoPlaylistObject
final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer"); .getObject(PLAYLIST_VIDEO_LIST_RENDERER)
final JsonArray videosArray = videos.getArray("contents"); .getArray("contents");
collectStreamsFrom(collector, videosArray); collectStreamsFrom(collector, videosArray);
nextPage = getNextPageFrom(videosArray); nextPage = getNextPageFrom(videosArray);
@ -229,7 +276,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0).getObject("appendContinuationItemsAction") .getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems"); .getArray("continuationItems");
collectStreamsFrom(collector, continuation); collectStreamsFrom(collector, continuation);
@ -237,8 +285,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
} }
private Page getNextPageFrom(final JsonArray contents) throws IOException, @Nullable
ExtractionException { private Page getNextPageFrom(final JsonArray contents)
throws IOException, ExtractionException {
if (isNullOrEmpty(contents)) { if (isNullOrEmpty(contents)) {
return null; return null;
} }
@ -252,10 +301,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getString("token"); .getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder( final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry()) getExtractorLocalization(), getExtractorContentCountry())
.value("continuation", continuation) .value("continuation", continuation)
.done()) .done())
.getBytes(UTF_8); .getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body); return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
} else { } else {
@ -263,20 +312,21 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
} }
private void collectStreamsFrom(final StreamInfoItemsCollector collector, private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
final JsonArray videos) { @Nonnull final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser(); final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object video : videos) { videos.stream()
if (((JsonObject) video).has("playlistVideoRenderer")) { .filter(JsonObject.class::isInstance)
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video) .map(JsonObject.class::cast)
.getObject("playlistVideoRenderer"), timeAgoParser) { .filter(video -> video.has(PLAYLIST_VIDEO_RENDERER))
.map(video -> new YoutubeStreamInfoItemExtractor(
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) {
@Override @Override
public long getViewCount() { public long getViewCount() {
return -1; return -1;
} }
}); })
} .forEachOrdered(collector::commit);
}
} }
} }

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -7,12 +7,14 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.*; import static org.schabi.newpipe.extractor.services.DefaultTests.*;
@ -85,9 +87,9 @@ public class SoundcloudPlaylistExtractorTest {
} }
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
// SoundCloud playlists do not have a banner // SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl()); assertEmpty(extractor.getBannerUrl());
} }
@Test @Test
@ -182,9 +184,9 @@ public class SoundcloudPlaylistExtractorTest {
} }
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
// SoundCloud playlists do not have a banner // SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl()); assertEmpty(extractor.getBannerUrl());
} }
@Test @Test
@ -294,9 +296,9 @@ public class SoundcloudPlaylistExtractorTest {
} }
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
// SoundCloud playlists do not have a banner // SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl()); assertEmpty(extractor.getBannerUrl());
} }
@Test @Test
@ -398,9 +400,9 @@ public class SoundcloudPlaylistExtractorTest {
} }
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
// SoundCloud playlists do not have a banner // SoundCloud playlists do not have a banner
assertNull(extractor.getBannerUrl()); assertEmpty(extractor.getBannerUrl());
} }
@Test @Test

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ListExtractor.ITEM_COUNT_UNKNOWN;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems; import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor; import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
@ -129,7 +130,7 @@ public class YoutubePlaylistExtractorTest {
@Disabled @Disabled
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
final String bannerUrl = extractor.getBannerUrl(); final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl); assertIsSecureUrl(bannerUrl);
ExtractorAsserts.assertContains("yt", bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl);
@ -249,7 +250,7 @@ public class YoutubePlaylistExtractorTest {
@Disabled @Disabled
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
final String bannerUrl = extractor.getBannerUrl(); final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl); assertIsSecureUrl(bannerUrl);
ExtractorAsserts.assertContains("yt", bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl);
@ -352,7 +353,7 @@ public class YoutubePlaylistExtractorTest {
@Disabled @Disabled
@Test @Test
public void testBannerUrl() { public void testBannerUrl() throws ParsingException {
final String bannerUrl = extractor.getBannerUrl(); final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl); assertIsSecureUrl(bannerUrl);
ExtractorAsserts.assertContains("yt", bannerUrl); ExtractorAsserts.assertContains("yt", bannerUrl);
@ -377,7 +378,8 @@ public class YoutubePlaylistExtractorTest {
@Test @Test
public void testStreamCount() throws Exception { public void testStreamCount() throws Exception {
ExtractorAsserts.assertGreater(40, extractor.getStreamCount()); // We are not able to extract the stream count of YouTube learning playlists
assertEquals(ITEM_COUNT_UNKNOWN, extractor.getStreamCount());
} }
@Override @Override