CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[YouTube] Detect deleted/nonexistent/invalid channels and playlists 

- Added tests for these cases.
This commit is contained in:
Mauricio Colli 2020-02-29 21:52:25 -03:00 committed by TobiGr
parent 98e359438a
commit e65333c3ce
5 changed files with 85 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -58,8 +58,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
@ -39,6 +40,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialData = ajaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);
playlistInfo = getPlaylistInfo();
}

35
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -362,10 +362,45 @@ public class YoutubeParsingHelper {
throw new ParsingException("JSON response is too short");
}
// Check if the request was redirected to the error page.
final URL latestUrl = new URL(response.latestUrl());
if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) {
final String path = latestUrl.getPath();
if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) {
throw new ContentNotAvailableException("Content unavailable");
}
}
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
}
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
/**
* Shared alert detection function, multiple endpoints return the error similarly structured.
* <p>
* Will check if the object has an alert of the type "ERROR".
*
* @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected
*/
public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
final JsonArray alerts = initialData.getArray("alerts");
if (alerts != null && !alerts.isEmpty()) {
final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
final String alertText = alertRenderer.getObject("text").getString("simpleText");
final String alertType = alertRenderer.getString("type");
if (alertType.equalsIgnoreCase("ERROR")) {
throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
}
}
}
}

23
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
@ -19,6 +20,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link ChannelExtractor}
*/
public class YoutubeChannelExtractorTest {
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ContentNotAvailableException.class)
public void deletedFetch() throws Exception {
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw");
extractor.fetchPage();
}
@Test(expected = ContentNotAvailableException.class)
public void nonExistentFetch() throws Exception {
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
extractor.fetchPage();
}
}
public static class Gronkh implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;

23
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
@ -23,6 +24,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link YoutubePlaylistExtractor}
*/
public class YoutubePlaylistExtractorTest {
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ContentNotAvailableException.class)
public void nonExistentFetch() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111");
extractor.fetchPage();
}
@Test(expected = ContentNotAvailableException.class)
public void invalidId() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");
extractor.fetchPage();
}
}
public static class TimelessPopHits implements BasePlaylistExtractorTest {
private static YoutubePlaylistExtractor extractor;