CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use pbj in YoutubeSearchExtractor

Browse source
This commit is contained in:
wb9688 2020-02-26 09:01:57 +01:00
parent fc465c8bf7
commit 0973263aab
1 changed files with 33 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View file

@ -47,8 +47,6 @@ import javax.annotation.Nonnull;
*/ */
public class YoutubeSearchExtractor extends SearchExtractor { public class YoutubeSearchExtractor extends SearchExtractor {
private Document doc;
private JsonObject initialData; private JsonObject initialData;
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
@ -57,10 +55,28 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl(); final String url = getUrl() + "&pbj=1";
final Response response = downloader.get(url, getExtractorLocalization());
doc = YoutubeParsingHelper.parseAndCheckPage(url, response); JsonArray ajaxJson;
initialData = YoutubeParsingHelper.getInitialData(response.responseBody());
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
try {
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json data for next streams", e);
}
initialData = ajaxJson.getObject(1).getObject("response");
} }
@Nonnull @Nonnull
@ -114,30 +130,22 @@ public class YoutubeSearchExtractor extends SearchExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
try { try {
// Use the hardcoded client version first to get JSON with a structure we know
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response); ajaxJson = JsonParser.array().from(response);
} catch (Exception e) { } catch (JsonParserException e) {
try { throw new ParsingException("Could not parse json data for next streams", e);
headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams");
}
ajaxJson = JsonParser.array().from(response);
} catch (JsonParserException ignored) {
throw new ParsingException("Could not parse json data for next streams", e);
}
} }
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response") JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation"); .getObject("continuationContents").getObject("itemSectionContinuation");