CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix:invidious: getID and onAccpetURl for comments

This commit is contained in:
B0pol 2020-01-17 12:09:59 +01:00
parent 4aaf123f24
commit a12c0e2a37
2 changed files with 157 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

208
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View File

@ -1,13 +1,15 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler; package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory { public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@ -18,75 +20,164 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
return instance; return instance;
} }
private static String assertIsID(String id) throws ParsingException {
if (id == null || !id.matches("[a-zA-Z0-9_-]{11}")) {
throw new ParsingException("The given string is not a Youtube-Video-ID");
}
return id;
}
@Override @Override
public String getUrl(String id) { public String getUrl(String id) {
return "https://m.youtube.com/watch?v=" + id; return "https://m.youtube.com/watch?v=" + id;
} }
@Override @Override
public String getId(String url) throws ParsingException, IllegalArgumentException { public String getId(String urlString) throws ParsingException, IllegalArgumentException {
if (url.isEmpty()) {
throw new IllegalArgumentException("The url parameter should not be empty");
}
String id;
String lowercaseUrl = url.toLowerCase();
if (lowercaseUrl.contains("youtube")) {
if (url.contains("attribution_link")) {
try { try {
String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url); URI uri = new URI(urlString);
String query = URLDecoder.decode(escapedQuery, "UTF-8"); String scheme = uri.getScheme();
id = Parser.matchGroup1("v=" + ID_PATTERN, query);
} catch (UnsupportedEncodingException uee) { if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
throw new ParsingException("Could not parse attribution_link", uee); String schemeSpecificPart = uri.getSchemeSpecificPart();
} if (schemeSpecificPart.startsWith("//")) {
} else if (url.contains("vnd.youtube")) { urlString = "https:" + schemeSpecificPart;
id = Parser.matchGroup1(ID_PATTERN, url);
} else if (url.contains("embed")) {
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
} else if (url.contains("googleads")) {
throw new FoundAdException("Error found add: " + url);
} else { } else {
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url); return assertIsID(schemeSpecificPart);
} }
} else if (lowercaseUrl.contains("youtu.be")) {
if (url.contains("v=")) {
id = Parser.matchGroup1("v=" + ID_PATTERN, url);
} else {
id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url);
} }
} else if(lowercaseUrl.contains("hooktube")) { } catch (URISyntaxException ignored) {
if(lowercaseUrl.contains("&v=")
|| lowercaseUrl.contains("?v=")) {
id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url);
} else if (url.contains("/embed/")) {
id = Parser.matchGroup1("embed/" + ID_PATTERN, url);
} else if (url.contains("/v/")) {
id = Parser.matchGroup1("v/" + ID_PATTERN, url);
} else if (url.contains("/watch/")) {
id = Parser.matchGroup1("watch/" + ID_PATTERN, url);
} else {
throw new ParsingException("Error no suitable url: " + url);
}
} else {
throw new ParsingException("Error no suitable url: " + url);
} }
URL url;
if (!id.isEmpty()) { try {
return id; url = Utils.stringToURL(urlString);
} else { } catch (MalformedURLException e) {
throw new ParsingException("Error could not parse url: " + url); throw new IllegalArgumentException("The given URL is not valid");
} }
String host = url.getHost();
String path = url.getPath();
// remove leading "/" of URL-path if URL-path is given
if (!path.isEmpty()) {
path = path.substring(1);
}
if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) ||
YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) ||
YoutubeParsingHelper.isInvidioURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString);
}
throw new ParsingException("The url is not a Youtube-URL");
}
if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) {
throw new ParsingException("Error no suitable url: " + urlString);
}
// using uppercase instead of lowercase, because toLowercase replaces some unicode characters
// with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls.
switch (host.toUpperCase()) {
case "WWW.YOUTUBE-NOCOOKIE.COM": {
if (path.startsWith("embed/")) {
String id = path.split("/")[1];
return assertIsID(id);
}
break;
}
case "YOUTUBE.COM":
case "WWW.YOUTUBE.COM":
case "M.YOUTUBE.COM":
case "MUSIC.YOUTUBE.COM": {
if (path.equals("attribution_link")) {
String uQueryValue = Utils.getQueryValue(url, "u");
URL decodedURL;
try {
decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue);
} catch (MalformedURLException e) {
throw new ParsingException("Error no suitable url: " + urlString);
}
String viewQueryValue = Utils.getQueryValue(decodedURL, "v");
return assertIsID(viewQueryValue);
}
if (path.startsWith("embed/")) {
String id = path.split("/")[1];
return assertIsID(id);
}
String viewQueryValue = Utils.getQueryValue(url, "v");
return assertIsID(viewQueryValue);
}
case "YOUTU.BE": {
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsID(viewQueryValue);
}
return assertIsID(path);
}
case "HOOKTUBE.COM": {
if (path.startsWith("v/")) {
String id = path.substring("v/".length());
return assertIsID(id);
}
if (path.startsWith("watch/")) {
String id = path.substring("watch/".length());
return assertIsID(id);
}
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
}
case "WWW.INVIDIO.US":
case "DEV.INVIDIO.US":
case "INVIDIO.US":
case "INVIDIOUS.SNOPYTA.ORG":
case "DE.INVIDIOUS.SNOPYTA.ORG":
case "FI.INVIDIOUS.SNOPYTA.ORG":
case "VID.WXZM.SX":
case "INVIDIOUS.KABI.TK":
case "INVIDIOU.SH":
case "WWW.INVIDIOU.SH":
case "NO.INVIDIOU.SH":
case "INVIDIOUS.ENKIRTON.NET":
case "TUBE.POAL.CO":
case "INVIDIOUS.13AD.DE":
case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances
if (path.equals("watch")) {
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
return assertIsID(viewQueryValue);
}
}
if (path.startsWith("embed/")) {
String id = path.substring("embed/".length());
return assertIsID(id);
}
break;
}
}
throw new ParsingException("Error no suitable url: " + urlString);
} }
@Override @Override
public boolean onAcceptUrl(final String url) throws FoundAdException { public boolean onAcceptUrl(final String url) throws FoundAdException {
final String lowercaseUrl = url.toLowerCase();
if (lowercaseUrl.contains("youtube")
|| lowercaseUrl.contains("youtu.be")
|| lowercaseUrl.contains("hooktube")) {
// bad programming I know
try { try {
getId(url); getId(url);
return true; return true;
@ -95,9 +186,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
} catch (ParsingException e) { } catch (ParsingException e) {
return false; return false;
} }
} else {
return false;
}
} }
@Override @Override

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -27,7 +27,7 @@ public class YoutubeCommentsExtractorTest {
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance()); NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeCommentsExtractor) YouTube extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor("https://www.youtube.com/watch?v=D00Au7k3i6o"); .getCommentsExtractor("https://www.invidio.us/watch?v=D00Au7k3i6o");
} }
@Test @Test
@ -47,7 +47,7 @@ public class YoutubeCommentsExtractorTest {
@Test @Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false; boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.youtube.com/watch?v=D00Au7k3i6o"); CommentsInfo commentsInfo = CommentsInfo.getInfo("https://www.invidio.us/watch?v=D00Au7k3i6o");
assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName())); assertTrue("what the fuck am i doing with my life".equals(commentsInfo.getName()));
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3"); result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");