CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use immutable sets in YoutubeParsingHelper.

Browse source
This commit is contained in:
Isira Seneviratne 2022-08-09 09:14:34 +05:30
parent eb07d70a2c
commit 316d8573fa
7 changed files with 33 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View file

@ -19,10 +19,10 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: set up JDK 8
- name: set up JDK 11
uses: actions/setup-java@v3
with:
java-version: '8'
java-version: '11'
distribution: 'temurin'
- name: Cache Gradle dependencies

4
build.gradle
View file

@ -5,8 +5,8 @@ allprojects {
compileJava.options.encoding = 'UTF-8'
compileTestJava.options.encoding = 'UTF-8'
sourceCompatibility = 1.8
targetCompatibility = 1.8
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_11
version 'v0.22.1'
group 'com.github.TeamNewPipe'

57
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file

@ -66,10 +66,12 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@ -241,6 +243,18 @@ public final class YoutubeParsingHelper {
private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
private static final Set<String> GOOGLE_URLS = Set.of("google.", "m.google.", "www.google.");
private static final Set<String> INVIDIOUS_URLS = Set.of("invidio.us", "dev.invidio.us",
"www.invidio.us", "redirect.invidious.io", "invidious.snopyta.org", "yewtu.be",
"tube.connect.cafe", "tubus.eduvid.org", "invidious.kavin.rocks", "invidious.site",
"invidious-us.kavin.rocks", "piped.kavin.rocks", "vid.mint.lgbt", "invidiou.site",
"invidious.fdn.fr", "invidious.048596.xyz", "invidious.zee.li", "vid.puffyan.us",
"ytprivate.com", "invidious.namazso.eu", "invidious.silkky.cloud", "ytb.trom.tf",
"invidious.exonip.de", "inv.riverside.rocks", "invidious.blamefran.net", "y.com.cm",
"invidious.moomoo.me", "yt.cyberhost.uk");
private static final Set<String> YOUTUBE_URLS = Set.of("youtube.com", "www.youtube.com",
"m.youtube.com", "music.youtube.com");
/**
* Determines how the consent cookie (that is required for YouTube) will be generated.
*
@ -262,21 +276,14 @@ public final class YoutubeParsingHelper {
final String cachedUrl = extractCachedUrlIfNeeded(url);
try {
final URL u = new URL(cachedUrl);
final String host = u.getHost();
return host.startsWith("google.")
|| host.startsWith("m.google.")
|| host.startsWith("www.google.");
return GOOGLE_URLS.stream().anyMatch(item -> u.getHost().startsWith(item));
} catch (final MalformedURLException e) {
return false;
}
}
public static boolean isYoutubeURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("youtube.com")
|| host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com")
|| host.equalsIgnoreCase("music.youtube.com");
return YOUTUBE_URLS.contains(url.getHost().toLowerCase(Locale.ROOT));
}
public static boolean isYoutubeServiceURL(@Nonnull final URL url) {
@ -290,36 +297,8 @@ public final class YoutubeParsingHelper {
return host.equalsIgnoreCase("hooktube.com");
}
public static boolean isInvidioURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("invidio.us")
|| host.equalsIgnoreCase("dev.invidio.us")
|| host.equalsIgnoreCase("www.invidio.us")
|| host.equalsIgnoreCase("redirect.invidious.io")
|| host.equalsIgnoreCase("invidious.snopyta.org")
|| host.equalsIgnoreCase("yewtu.be")
|| host.equalsIgnoreCase("tube.connect.cafe")
|| host.equalsIgnoreCase("tubus.eduvid.org")
|| host.equalsIgnoreCase("invidious.kavin.rocks")
|| host.equalsIgnoreCase("invidious-us.kavin.rocks")
|| host.equalsIgnoreCase("piped.kavin.rocks")
|| host.equalsIgnoreCase("invidious.site")
|| host.equalsIgnoreCase("vid.mint.lgbt")
|| host.equalsIgnoreCase("invidiou.site")
|| host.equalsIgnoreCase("invidious.fdn.fr")
|| host.equalsIgnoreCase("invidious.048596.xyz")
|| host.equalsIgnoreCase("invidious.zee.li")
|| host.equalsIgnoreCase("vid.puffyan.us")
|| host.equalsIgnoreCase("ytprivate.com")
|| host.equalsIgnoreCase("invidious.namazso.eu")
|| host.equalsIgnoreCase("invidious.silkky.cloud")
|| host.equalsIgnoreCase("invidious.exonip.de")
|| host.equalsIgnoreCase("inv.riverside.rocks")
|| host.equalsIgnoreCase("invidious.blamefran.net")
|| host.equalsIgnoreCase("invidious.moomoo.me")
|| host.equalsIgnoreCase("ytb.trom.tf")
|| host.equalsIgnoreCase("yt.cyberhost.uk")
|| host.equalsIgnoreCase("y.com.cm");
public static boolean isInvidiousURL(@Nonnull final URL url) {
return INVIDIOUS_URLS.contains(url.getHost().toLowerCase(Locale.ROOT));
}
public static boolean isY2ubeURL(@Nonnull final URL url) {

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View file

@ -20,7 +20,6 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.util.regex.Pattern;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@ -29,6 +28,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.net.URL;
import java.util.List;
import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
@ -90,7 +90,7 @@ public final class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFacto
String path = urlObj.getPath();
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj)
|| YoutubeParsingHelper.isInvidiousURL(urlObj)
|| YoutubeParsingHelper.isHooktubeURL(urlObj))) {
throw new ParsingException("The URL given is not a YouTube URL");
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java
View file

@ -1,8 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
@ -11,6 +8,10 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubePlaylistLinkHandlerFactory INSTANCE =
@ -35,7 +36,7 @@ public final class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFact
final URL urlObj = Utils.stringToURL(url);
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj))) {
|| YoutubeParsingHelper.isInvidiousURL(urlObj))) {
throw new ParsingException("the url given is not a YouTube-URL");
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View file

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isHooktubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isY2ubeURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
@ -122,7 +122,7 @@ public final class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
}
if (!Utils.isHTTP(url) || !(isYoutubeURL(url) || isYoutubeServiceURL(url)
|| isHooktubeURL(url) || isInvidioURL(url) || isY2ubeURL(url))) {
|| isHooktubeURL(url) || isInvidiousURL(url) || isY2ubeURL(url))) {
if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) {
throw new FoundAdException("Error found ad: " + urlString);
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java
View file

@ -20,7 +20,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidiousURL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@ -53,7 +53,7 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
}
final String urlPath = urlObj.getPath();
return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj))
return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidiousURL(urlObj))
&& urlPath.equals("/feed/trending");
}
}