CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #580 from TeamNewPipe/accountTerminated

Browse source 
Add AccountTerminatedException for terminated channels
This commit is contained in:
Tobi 2021-06-08 09:55:58 +02:00 committed by GitHub
commit d4186d100b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 715 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/AccountTerminatedException.java Normal file
View file

@ -0,0 +1,31 @@
package org.schabi.newpipe.extractor.exceptions;
public class AccountTerminatedException extends ContentNotAvailableException {
private Reason reason = Reason.UNKNOWN;
public AccountTerminatedException(final String message) {
super(message);
}
public AccountTerminatedException(final String message, final Reason reason) {
super(message);
this.reason = reason;
}
public AccountTerminatedException(final String message, final Throwable cause) {
super(message, cause);
}
/**
* The reason for the violation. There should also be more info in the exception's message.
*/
public Reason getReason() {
return reason;
}
public enum Reason {
UNKNOWN,
VIOLATION
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file

@ -10,10 +10,7 @@ import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -762,6 +759,23 @@ public class YoutubeParsingHelper {
final String alertText = getTextFromObject(alertRenderer.getObject("text"));
final String alertType = alertRenderer.getString("type", EMPTY_STRING);
if (alertType.equalsIgnoreCase("ERROR")) {
if (alertText != null && alertText.contains("This account has been terminated")) {
if (alertText.contains("violation") || alertText.contains("violating")
|| alertText.contains("infringement")) {
// possible error messages:
// "This account has been terminated for a violation of YouTube's Terms of Service."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech."
// "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten."
// "This account has been terminated due to multiple or severe violations of YouTube's policy against spam, deceptive practices and misleading content or other Terms of Service violations."
// "This account has been terminated due to multiple or severe violations of YouTube's policy on nudity or sexual content."
// "This account has been terminated for violating YouTube's Community Guidelines."
// "This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted."
// "This account has been terminated because it is linked to an account that received multiple third-party claims of copyright infringement."
throw new AccountTerminatedException(alertText, AccountTerminatedException.Reason.VIOLATION);
} else {
throw new AccountTerminatedException(alertText);
}
}
throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java
View file

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -33,6 +34,9 @@ public class YoutubeFeedExtractor extends FeedExtractor {
final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
final Response response = downloader.get(feedUrl);
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Could not get feed: 404 - not found");
}
document = Jsoup.parse(response.responseBody());
}

85
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View file

@ -6,6 +6,7 @@ import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -50,6 +51,90 @@ public class YoutubeChannelExtractorTest {
YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
extractor.fetchPage();
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedTOSFetch() throws Exception {
// "This account has been terminated for a violation of YouTube's Terms of Service."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedCommunityFetch() throws Exception {
// "This account has been terminated for violating YouTube's Community Guidelines."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedHateFetch() throws Exception {
// "This account has been terminated due to multiple or severe violations
// of YouTube's policy prohibiting hate speech."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCPWXIOPK-9myzek6jHR5yrg");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedBullyFetch() throws Exception {
// "This account has been terminated due to multiple or severe violations
// of YouTube's policy prohibiting content designed to harass, bully or threaten."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedSpamFetch() throws Exception {
// "This account has been terminated due to multiple or severe violations
// of YouTube's policy against spam, deceptive practices and misleading content
// or other Terms of Service violations."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCoaO4U_p7G7AwalqSbGCZOA");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
@Test(expected = AccountTerminatedException.class)
public void accountTerminatedCopyrightFetch() throws Exception {
// "This account has been terminated because we received multiple third-party claims
// of copyright infringement regarding material that the user posted."
final ChannelExtractor extractor =
YouTube.getChannelExtractor("https://www.youtube.com/channel/UCpExuV8qJMfCaSQNL1YG6bQ");
try {
extractor.fetchPage();
} catch (AccountTerminatedException e) {
assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
throw e;
}
}
}
public static class NotSupported {

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java
View file

@ -4,10 +4,12 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import java.io.IOException;
import java.util.Random;
import static org.junit.Assert.assertEquals;
@ -77,4 +79,20 @@ public class YoutubeFeedExtractorTest {
assertNoMoreItems(extractor);
}
}
public static class NotAvailable {
@BeforeClass
public static void setUp() throws IOException {
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable/"));
}
@Test(expected = ContentNotAvailableException.class)
public void AccountTerminatedFetch() throws Exception {
YoutubeFeedExtractor extractor = (YoutubeFeedExtractor) YouTube
.getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
extractor.fetchPage();
}
}
}

11
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_0.json
View file

File diff suppressed because one or more lines are too long

15
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_1.json
View file

File diff suppressed because one or more lines are too long

24
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_2.json
View file

File diff suppressed because one or more lines are too long

81
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_3.json Normal file
View file

File diff suppressed because one or more lines are too long

81
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_4.json Normal file
View file

File diff suppressed because one or more lines are too long

81
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_5.json Normal file
View file

File diff suppressed because one or more lines are too long

81
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_6.json Normal file
View file

File diff suppressed because one or more lines are too long

81
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_7.json Normal file
View file

File diff suppressed because one or more lines are too long

74
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_8.json Normal file
View file

@ -0,0 +1,74 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"Cookie": [
"CONSENT\u003dPENDING+100406"
],
"X-YouTube-Client-Name": [
"1"
],
"X-YouTube-Client-Version": [
"2.20200214.04.00"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 404,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
],
"content-type": [
"text/html; charset\u003dutf-8"
],
"date": [
"Sat, 01 May 2021 15:47:25 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
],
"pragma": [
"no-cache"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dgPx3EMoLzV4; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"J6HBrtOLDsO+k15HZ0o11w\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
}
}

9
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/generated_mock_0.json
View file

File diff suppressed because one or more lines are too long

44
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/notAvailable/generated_mock_0.json Normal file
View file

@ -0,0 +1,44 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 404,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"content-length": [
"1613"
],
"content-type": [
"text/html; charset\u003dUTF-8"
],
"date": [
"Tue, 30 Mar 2021 08:33:07 GMT"
],
"server": [
"YouTube RSS Feeds server"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\u003c!DOCTYPE html\u003e\n\u003chtml lang\u003den\u003e\n \u003cmeta charset\u003dutf-8\u003e\n \u003cmeta name\u003dviewport content\u003d\"initial-scale\u003d1, minimum-scale\u003d1, width\u003ddevice-width\"\u003e\n \u003ctitle\u003eError 404 (Not Found)!!1\u003c/title\u003e\n \u003cstyle\u003e\n *{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px}* \u003e body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}\n \u003c/style\u003e\n \u003ca href\u003d//www.google.com/\u003e\u003cspan id\u003dlogo aria-label\u003dGoogle\u003e\u003c/span\u003e\u003c/a\u003e\n \u003cp\u003e\u003cb\u003e404.\u003c/b\u003e \u003cins\u003eThat’s an error.\u003c/ins\u003e\n \u003cp\u003eThe requested URL \u003ccode\u003e/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ\u003c/code\u003e was not found on this server. \u003cins\u003eThat’s all we know.\u003c/ins\u003e\n",
"latestUrl": "https://www.youtube.com/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ"
}
}