CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adress changes

This commit is contained in:
TiA4f8R 2021-06-24 18:39:16 +02:00
parent 1a6b8da438
commit 4299d806a2
No known key found for this signature in database
GPG Key ID: E6D3E7F5949450DD
7 changed files with 95 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -66,15 +66,15 @@ public class YoutubeParsingHelper {
public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
private static final String HARDCODED_CLIENT_VERSION = "2.20210603.07.00";
private static final String HARDCODED_CLIENT_VERSION = "2.20210622.10.00";
private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.20.36";
private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.23.36";
private static String clientVersion;
private static String key;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY =
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
{"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210621.00.00"};
private static String[] youtubeMusicKey;
private static boolean keyAndVersionExtracted = false;
@ -102,7 +102,8 @@ public class YoutubeParsingHelper {
try {
final URL u = new URL(url);
final String host = u.getHost();
return host.startsWith("google.") || host.startsWith("m.google.")
return host.startsWith("google.")
|| host.startsWith("m.google.")
|| host.startsWith("www.google.");
} catch (final MalformedURLException e) {
return false;
@ -111,7 +112,8 @@ public class YoutubeParsingHelper {
public static boolean isYoutubeURL(@Nonnull final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
return host.equalsIgnoreCase("youtube.com")
|| host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com")
|| host.equalsIgnoreCase("music.youtube.com");
}
@ -234,7 +236,7 @@ public class YoutubeParsingHelper {
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
*
* @param playlistId the id of the playlist
* @param playlistId the playlist id
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
@ -306,8 +308,8 @@ public class YoutubeParsingHelper {
}
}
public static boolean areHardcodedClientVersionAndKeyValid() throws IOException,
ExtractionException {
public static boolean areHardcodedClientVersionAndKeyValid()
throws IOException, ExtractionException {
if (areHardcodedClientVersionAndKeyValidValue != null) {
return areHardcodedClientVersionAndKeyValidValue;
}
@ -316,11 +318,17 @@ public class YoutubeParsingHelper {
.object()
.object("context")
.object("client")
.value("hl", "en")
.value("hl", "en-GB")
.value("gl", "GB")
.value("clientName", "1")
.value("clientName", "WEB")
.value("clientVersion", HARDCODED_CLIENT_VERSION)
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.value("fetchLiveState", true)
.end()
.end().done().getBytes(UTF_8);
// @formatter:on
@ -337,9 +345,8 @@ public class YoutubeParsingHelper {
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();
areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000
return areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000
&& responseCode == 200; // Ensure to have a valid response
return areHardcodedClientVersionAndKeyValidValue;
}
private static void extractClientVersionAndKey() throws IOException, ExtractionException {
@ -485,8 +492,7 @@ public class YoutubeParsingHelper {
.value("hl", "en-GB")
.value("gl", "GB")
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.value("experimentsToken", EMPTY_STRING)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
@ -802,10 +808,13 @@ public class YoutubeParsingHelper {
return JsonObject.builder()
.object("context")
.object("client")
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("clientName", "WEB")
.value("clientVersion", getClientVersion())
.end()
.object("user")
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
@ -826,6 +835,11 @@ public class YoutubeParsingHelper {
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
// .value("enableSafetyMode", boolean)
.value("lockedSafetyMode", false)
.end()
.end();
// @formatter:on
}
@ -848,15 +862,11 @@ public class YoutubeParsingHelper {
*/
public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>> headers)
throws IOException, ExtractionException {
if (headers.get("Origin") == null) {
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
}
if (headers.get("Referer") == null) {
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
}
if (headers.get("X-YouTube-Client-Name") == null) {
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
}
headers.computeIfAbsent("Origin", k -> Collections.singletonList(
"https://www.youtube.com"));
headers.computeIfAbsent("Referer", k -> Collections.singletonList(
"https://www.youtube.com"));
headers.computeIfAbsent("X-YouTube-Client-Name", k -> Collections.singletonList("1"));
if (headers.get("X-YouTube-Client-Version") == null) {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
}
@ -867,7 +877,7 @@ public class YoutubeParsingHelper {
* @see #CONSENT_COOKIE
* @param headers the headers which should be completed
*/
public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
public static void addCookieHeader(final Map<String, List<String>> headers) {
if (headers.get("Cookie") == null) {
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
} else {
@ -1092,5 +1102,4 @@ public class YoutubeParsingHelper {
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
}
}

42
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
@ -84,7 +85,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String[] channelInfo = channel_path.split("/");
String id = "";
// If the url is an URL which is not a /channel URL, we need to use the
// navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we
// navigation/resolve_url endpoint of the internal API to get the channel id. Otherwise, we
// couldn't get information about the channel associated with this URL, if there is one.
if (!channelInfo[0].equals("channel")) {
final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),
@ -293,17 +294,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
}
@Override
public String getParentChannelName() throws ParsingException {
public String getParentChannelName() {
return "";
}
@Override
public String getParentChannelUrl() throws ParsingException {
public String getParentChannelUrl() {
return "";
}
@Override
public String getParentChannelAvatarUrl() throws ParsingException {
public String getParentChannelAvatarUrl() {
return "";
}
@ -329,13 +330,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer");
final List<String> channelInformations = new ArrayList<>();
channelInformations.add(getName());
channelInformations.add(getUrl());
final List<String> channelInfo = new ArrayList<>();
channelInfo.add(getName());
channelInfo.add(getUrl());
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer
.getArray("items"), channelInformations);
.getArray("items"), channelInfo);
nextPage = getNextPageFrom(continuation, channelInformations);
nextPage = getNextPageFrom(continuation, channelInfo);
}
return new InfoItemsPage<>(collector, nextPage);
@ -348,7 +349,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final List<String> channelInformations = page.getIds();
final List<String> channelInfos = page.getIds();
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final Map<String, List<String>> headers = new HashMap<>();
@ -364,13 +365,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation
.getArray("continuationItems"), channelInformations);
.getArray("continuationItems"), channelInfos);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInformations));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInfos));
}
@Nullable
private Page getNextPageFrom(final JsonObject continuations,
final List<String> channelInformations) throws IOException,
final List<String> channelInfo) throws IOException,
ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
@ -386,8 +388,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.done())
.getBytes(UTF_8);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInformations,
null, body);
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInfo, null, body);
}
/**
@ -397,13 +398,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
* @param videos the array to get videos from
* @return the continuation object
*/
private JsonObject collectStreamsFrom(final StreamInfoItemsCollector collector,
final JsonArray videos,
final List<String> channelInformations) {
private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
@Nonnull final JsonArray videos,
@Nonnull final List<String> channelInfo) {
collector.reset();
final String uploaderName = channelInformations.get(0);
final String uploaderUrl = channelInformations.get(1);
final String uploaderName = channelInfo.get(0);
final String uploaderUrl = channelInfo.get(1);
final TimeAgoParser timeAgoParser = getTimeAgoParser();
JsonObject continuation = null;
@ -431,6 +432,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return continuation;
}
@Nullable
private JsonObject getVideoTab() throws ParsingException {
if (this.videoTab != null) return this.videoTab;

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonBuilder;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
@ -61,31 +62,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
final String videoId = getQueryValue(url, "v");
final String playlistIndexString = getQueryValue(url, "index");
final byte[] body;
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
getExtractorContentCountry()).value("playlistId", mixPlaylistId);
if (videoId != null) {
if (playlistIndexString != null) {
body = JsonWriter.string(prepareJsonBuilder(localization,
getExtractorContentCountry())
.value("videoId", videoId)
.value("playlistId", mixPlaylistId)
.value("playlistIndex", Integer.parseInt(playlistIndexString))
.done())
.getBytes(UTF_8);
} else {
body = JsonWriter.string(prepareJsonBuilder(localization,
getExtractorContentCountry())
.value("videoId", videoId)
.value("playlistId", mixPlaylistId)
.done())
.getBytes(UTF_8);
}
} else {
body = JsonWriter.string(prepareJsonBuilder(localization,
getExtractorContentCountry())
.value("playlistId", mixPlaylistId)
.done())
.getBytes(UTF_8);
jsonBody.value("videoId", videoId);
}
if (playlistIndexString != null) {
jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString));
}
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -74,11 +74,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en")
.value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.value("experimentsToken", EMPTY_STRING)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
@ -89,6 +88,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end()
.object("activePlayers").end()
.object("user")
// TO DO: provide a way to enable restricted mode with:
.value("enableSafetyMode", false)
.end()
.end()

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
@ -60,7 +59,6 @@ public class YoutubeSearchExtractor extends SearchExtractor {
ExtractionException {
final String query = super.getSearchString();
final Localization localization = getExtractorLocalization();
final ContentCountry contentCountry = getExtractorContentCountry();
// Get the search parameter of the request
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
@ -72,20 +70,15 @@ public class YoutubeSearchExtractor extends SearchExtractor {
params = "";
}
final byte[] body;
final JsonBuilder<JsonObject> jsonBody = prepareJsonBuilder(localization,
getExtractorContentCountry())
.value("query", query);
if (!isNullOrEmpty(params)) {
body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
.value("query", query)
.value("params", params)
.done())
.getBytes(UTF_8);
} else {
body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry)
.value("query", query)
.done())
.getBytes(UTF_8);
jsonBody.value("params", params);
}
final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8);
initialData = getJsonPostResponse("search", body, localization);
}

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -482,6 +482,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getDashMpdUrl() throws ParsingException {
assertPageFetched();
try {
String dashManifestUrl;
if (streamingData.isString("dashManifestUrl")) {
@ -643,7 +644,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public StreamType getStreamType() {
assertPageFetched();
return streamingData.has(FORMATS) ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM;
if (playerResponse.getObject("videoDetails").getBoolean("isLiveContent", false)) {
return StreamType.LIVE_STREAM;
}
return StreamType.VIDEO_STREAM;
}
@Nullable
@ -895,7 +900,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
// The JavaScript player was not found in any page fetched so far and there is
// nothing cached, so try fetching embedded info.
// Don't provide a video id to get a smaller response (around 9kb instead of 21 kb
// Don't provide a video id to get a smaller response (around 9Kb instead of 21 Kb
// with a video)
final String embedUrl = "https://www.youtube.com/embed/";
final String embedPageContent = NewPipe.getDownloader()
@ -936,23 +941,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private boolean hasOtfStreams() {
if (streamingData != null) {
boolean hasOtfStreamsValue = false;
if (streamingData.has("adaptiveFormats")) {
final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
for (final Object adaptiveFormat : adaptiveFormats) {
final JsonObject jsonAdaptiveFormat = (JsonObject) adaptiveFormat;
if (jsonAdaptiveFormat.has("type")) {
final String streamTypeFormat = jsonAdaptiveFormat.getString("type",
EMPTY_STRING);
if (streamTypeFormat.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
hasOtfStreamsValue = true;
break;
}
}
final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats");
for (final Object adaptiveFormat : adaptiveFormats) {
final JsonObject jsonAdaptiveFormat = (JsonObject) adaptiveFormat;
final String streamTypeFormat = jsonAdaptiveFormat.getString("type", EMPTY_STRING);
if (streamTypeFormat.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
return true;
}
}
return hasOtfStreamsValue;
}
return false;
}
@ -1123,9 +1121,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
private static String getVideoInfoUrl(final String id, final String sts) {
// TODO: Try parsing embedded_player_response first
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
"&sts=" + sts + "&ps=default&gl=US&hl=en";
return "https://www.youtube.com/get_video_info?" + "video_id=" + id
+ "&eurl=https://youtube.googleapis.com/v/" + id + "&sts=" + sts
+ "&html5=1&c=TVHTML5&cver=6.20180913&hl=en&gl=US";
}
@Nonnull
@ -1280,7 +1278,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getCategory() {
return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer")
.getString("category");
.getString("category", EMPTY_STRING);
}
@Nonnull

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -58,11 +58,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Override public int expectedAgeLimit() { return 18; }
@Override public boolean expectedHasSubtitles() { return false; }
<<<<<<< HEAD
@Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos
=======
@Override public String expectedCategory() { return "Entertainment"; }
>>>>>>> d0dc7d69 (Update mocks, reenable a test and fix a test)
@Override public String expectedLicence() { return "YouTube licence"; }
@Override
public List<String> expectedTags() {