CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use pbj in YoutubePlaylistExtractor

Browse source
This commit is contained in:
wb9688 2020-02-26 14:45:50 +01:00
parent 89a41a7f69
commit 15afbea3e1
1 changed files with 33 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -5,10 +5,8 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -29,8 +27,6 @@ import javax.annotation.Nonnull;
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor { public class YoutubePlaylistExtractor extends PlaylistExtractor {
private Document doc;
private JsonObject initialData; private JsonObject initialData;
private JsonObject uploaderInfo; private JsonObject uploaderInfo;
private JsonObject playlistInfo; private JsonObject playlistInfo;
@ -41,10 +37,28 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl(); final String url = getUrl() + "&pbj=1";
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response); JsonArray ajaxJson;
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
try {
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
initialData = ajaxJson.getObject(1).getObject("response");
uploaderInfo = getUploaderInfo(); uploaderInfo = getUploaderInfo();
playlistInfo = getPlaylistInfo(); playlistInfo = getPlaylistInfo();
} }
@ -188,28 +202,20 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
try {
// Use the hardcoded client version first to get JSON with a structure we know // Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");
} }
ajaxJson = JsonParser.array().from(response);
} catch (Exception e) {
try { try {
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response); ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) { } catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e); throw new ParsingException("Could not parse json data for next streams", e);
} }
}
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation"); .getObject("continuationContents").getObject("playlistVideoListContinuation");