[YouTube] Detect deleted/nonexistent/invalid channels and playlists

- Added tests for these cases.
This commit is contained in:
Mauricio Colli 2020-02-29 21:52:25 -03:00 committed by TobiGr
parent 98e359438a
commit e65333c3ce
5 changed files with 85 additions and 1 deletions

View File

@ -58,8 +58,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response"); initialData = ajaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);
} }

View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -39,6 +40,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response"); initialData = ajaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo(); playlistInfo = getPlaylistInfo();
} }

View File

@ -362,10 +362,45 @@ public class YoutubeParsingHelper {
throw new ParsingException("JSON response is too short"); throw new ParsingException("JSON response is too short");
} }
// Check if the request was redirected to the error page.
final URL latestUrl = new URL(response.latestUrl());
if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) {
final String path = latestUrl.getPath();
if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) {
throw new ContentNotAvailableException("Content unavailable");
}
}
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
}
try { try {
return JsonParser.array().from(responseBody); return JsonParser.array().from(responseBody);
} catch (JsonParserException e) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e); throw new ParsingException("Could not parse JSON", e);
} }
} }
/**
* Shared alert detection function, multiple endpoints return the error similarly structured.
* <p>
* Will check if the object has an alert of the type "ERROR".
*
* @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected
*/
public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
final JsonArray alerts = initialData.getArray("alerts");
if (alerts != null && !alerts.isEmpty()) {
final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
final String alertText = alertRenderer.getObject("text").getString("simpleText");
final String alertType = alertRenderer.getString("type");
if (alertType.equalsIgnoreCase("ERROR")) {
throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
}
}
}
} }

View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
@ -19,6 +20,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link ChannelExtractor} * Test for {@link ChannelExtractor}
*/ */
public class YoutubeChannelExtractorTest { public class YoutubeChannelExtractorTest {
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ContentNotAvailableException.class)
public void deletedFetch() throws Exception {
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw");
extractor.fetchPage();
}
@Test(expected = ContentNotAvailableException.class)
public void nonExistentFetch() throws Exception {
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
extractor.fetchPage();
}
}
public static class Gronkh implements BaseChannelExtractorTest { public static class Gronkh implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor; private static YoutubeChannelExtractor extractor;

View File

@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
@ -23,6 +24,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link YoutubePlaylistExtractor} * Test for {@link YoutubePlaylistExtractor}
*/ */
public class YoutubePlaylistExtractorTest { public class YoutubePlaylistExtractorTest {
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ContentNotAvailableException.class)
public void nonExistentFetch() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111");
extractor.fetchPage();
}
@Test(expected = ContentNotAvailableException.class)
public void invalidId() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");
extractor.fetchPage();
}
}
public static class TimelessPopHits implements BasePlaylistExtractorTest { public static class TimelessPopHits implements BasePlaylistExtractorTest {
private static YoutubePlaylistExtractor extractor; private static YoutubePlaylistExtractor extractor;