CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[YouTube] Fix extraction of series playlists and don't return the view count as the stream count for learning playlists

Browse source 
ITEM_COUNT_UNKNOWN is returned when the JSON array which contains usally the number of videos is less than 3 items.
Also apply the same type of optimizations done in other PlaylistExtractors in YoutubePlaylistExtractor.
This commit is contained in:
TiA4f8R 2022-03-12 17:28:36 +01:00
parent 58a247907e
commit 8b3f90eb7e
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
1 changed files with 122 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

197
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -21,22 +21,31 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.*;
@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonObject initialData;
// Minimum size of the stats array in the browse response which includes the streams count
private static final int STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE = 2;
// Names of some objects in JSON response frequently used in this class
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
private JsonObject browseResponse;
private JsonObject playlistInfo;
public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
public YoutubePlaylistExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@ -45,41 +54,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
ExtractionException {
final Localization localization = getExtractorLocalization();
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
getExtractorContentCountry())
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // Show unavailable videos
.done())
.getBytes(UTF_8);
getExtractorContentCountry())
.value("browseId", "VL" + getId())
.value("params", "wgYCCAA%3D") // Show unavailable videos
.done())
.getBytes(StandardCharsets.UTF_8);
initialData = getJsonPostResponse("browse", body, localization);
YoutubeParsingHelper.defaultAlertsCheck(initialData);
browseResponse = getJsonPostResponse("browse", body, localization);
YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
playlistInfo = getPlaylistInfo();
}
private JsonObject getUploaderInfo() throws ParsingException {
final JsonArray items = initialData.getObject("sidebar")
.getObject("playlistSidebarRenderer").getArray("items");
final JsonArray items = browseResponse.getObject("sidebar")
.getObject("playlistSidebarRenderer")
.getArray("items");
JsonObject videoOwner = items.getObject(1)
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
.getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner");
if (videoOwner.has(VIDEO_OWNER_RENDERER)) {
return videoOwner.getObject(VIDEO_OWNER_RENDERER);
}
// we might want to create a loop here instead of using duplicated code
videoOwner = items.getObject(items.size())
.getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
return videoOwner.getObject("videoOwnerRenderer");
.getObject("playlistSidebarSecondaryInfoRenderer")
.getObject("videoOwner");
if (videoOwner.has(VIDEO_OWNER_RENDERER)) {
return videoOwner.getObject(VIDEO_OWNER_RENDERER);
}
throw new ParsingException("Could not get uploader info");
}
private JsonObject getPlaylistInfo() throws ParsingException {
try {
return initialData.getObject("sidebar").getObject("playlistSidebarRenderer")
.getArray("items").getObject(0)
return browseResponse.getObject("sidebar")
.getObject("playlistSidebarRenderer")
.getArray("items")
.getObject(0)
.getObject("playlistSidebarPrimaryInfoRenderer");
} catch (final Exception e) {
throw new ParsingException("Could not get PlaylistInfo", e);
@ -90,33 +104,41 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title"));
if (!isNullOrEmpty(name)) return name;
if (!isNullOrEmpty(name)) {
return name;
}
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
return browseResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getString("title");
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
String url = playlistInfo.getObject("thumbnailRenderer").getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
String url = playlistInfo.getObject("thumbnailRenderer")
.getObject("playlistVideoThumbnailRenderer")
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
if (isNullOrEmpty(url)) {
url = initialData.getObject("microformat").getObject("microformatDataRenderer").getObject("thumbnail")
.getArray("thumbnails").getObject(0).getString("url");
url = browseResponse.getObject("microformat")
.getObject("microformatDataRenderer")
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
if (isNullOrEmpty(url)) throw new ParsingException("Could not get playlist thumbnail");
if (isNullOrEmpty(url)) {
throw new ParsingException("Could not get playlist thumbnail");
}
}
return fixThumbnailUrl(url);
}
@Override
public String getBannerUrl() {
// Banner can't be handled by frontend right now.
// Whoever is willing to implement this should also implement it in the frontend.
return "";
}
@Override
public String getUploaderUrl() throws ParsingException {
try {
@ -138,7 +160,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
final String url = getUploaderInfo()
.getObject("thumbnail")
.getArray("thumbnails")
.getObject(0)
.getString("url");
return fixThumbnailUrl(url);
} catch (final Exception e) {
@ -148,14 +174,29 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public boolean isUploaderVerified() throws ParsingException {
// YouTube doesn't provide this information
return false;
}
@Override
public long getStreamCount() throws ParsingException {
try {
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
final JsonArray stats = playlistInfo.getArray("stats");
// For unknown reasons, YouTube don't provide the stream count for learning playlists
// on the desktop client but only the number of views and the playlist modified date
// On normal playlists, at least 3 items are returned: the number of videos, the number
// of views and the playlist modification date
// We can get it by using another client, however it seems we can't get the avatar
// uploader URL with another client than the WEB client
if (stats.size() > STATS_ARRAY_WITH_STREAMS_COUNT_MIN_SIZE) {
final String videosText = getTextFromObject(playlistInfo.getArray("stats")
.getObject(0));
if (videosText != null) {
return Long.parseLong(Utils.removeNonDigitCharacters(videosText));
}
}
return ITEM_COUNT_UNKNOWN;
} catch (final Exception e) {
throw new ParsingException("Could not get video count from playlist", e);
}
@ -164,19 +205,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public String getSubChannelName() {
return "";
return EMPTY_STRING;
}
@Nonnull
@Override
public String getSubChannelUrl() {
return "";
return EMPTY_STRING;
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() {
return "";
return EMPTY_STRING;
}
@Nonnull
@ -185,26 +226,31 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
final JsonArray contents = initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer").getArray("tabs").getObject(0)
.getObject("tabRenderer").getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
final JsonArray contents = browseResponse.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
.getObject(0)
.getObject("tabRenderer")
.getObject("content")
.getObject("sectionListRenderer")
.getArray("contents");
if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer")
.has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment)
.getObject("playlistSegmentRenderer").getObject("videoList")
.getObject("playlistVideoListRenderer").getArray("contents"));
}
}
final JsonObject videoPlaylistObject = contents.stream()
.map(JsonObject.class::cast)
.map(content -> content.getObject("itemSectionRenderer")
.getArray("contents")
.getObject(0))
.filter(contentItemSectionRendererContents ->
contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
|| contentItemSectionRendererContents.has(
"playlistSegmentRenderer"))
.findFirst()
.orElse(null);
return new InfoItemsPage<>(collector, null);
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer");
final JsonArray videosArray = videos.getArray("contents");
if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
final JsonArray videosArray = videoPlaylistObject
.getObject(PLAYLIST_VIDEO_LIST_RENDERER)
.getArray("contents");
collectStreamsFrom(collector, videosArray);
nextPage = getNextPageFrom(videosArray);
@ -229,7 +275,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0).getObject("appendContinuationItemsAction")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
collectStreamsFrom(collector, continuation);
@ -237,8 +284,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) throws IOException,
ExtractionException {
@Nullable
private Page getNextPageFrom(final JsonArray contents)
throws IOException, ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
@ -252,10 +300,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getString("token");
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
getExtractorLocalization(), getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(UTF_8);
getExtractorLocalization(), getExtractorContentCountry())
.value("continuation", continuation)
.done())
.getBytes(StandardCharsets.UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body);
} else {
@ -263,20 +311,19 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonArray videos) {
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (final Object video : videos) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video)
.getObject("playlistVideoRenderer"), timeAgoParser) {
videos.stream()
.filter(video -> ((JsonObject) video).has(PLAYLIST_VIDEO_RENDERER))
.map(video -> new YoutubeStreamInfoItemExtractor(((JsonObject) video)
.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser) {
@Override
public long getViewCount() {
return -1;
}
});
}
}
})
.forEachOrdered(collector::commit);
}
}