CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

using mobile website since it is faster

Browse source
This commit is contained in:
Ritvik Saraf 2018-09-28 04:54:57 +05:30
parent c2ed99b1b7
commit d1ff1c7589
3 changed files with 41 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View file

@ -8,6 +8,8 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.annotation.Nonnull;
import org.schabi.newpipe.extractor.DownloadRequest; import org.schabi.newpipe.extractor.DownloadRequest;
import org.schabi.newpipe.extractor.DownloadResponse; import org.schabi.newpipe.extractor.DownloadResponse;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
@ -26,14 +28,12 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
public class YoutubeCommentsExtractor extends CommentsExtractor { public class YoutubeCommentsExtractor extends CommentsExtractor {
private static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0"; private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0";
private List<String> cookies;
private String sessionToken;
private String ytClientVersion; private String ytClientVersion;
private String ytClientName; private String ytClientName;
private String title; private String title;
@ -45,17 +45,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override @Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
// initial page does not load any comments but is required to get session token // initial page does not load any comments but is required to get comments token
// and cookies
super.fetchPage(); super.fetchPage();
return initPage; return initPage;
} }
// isn't this method redundant. you can just call getnextpage on getInitialPage
@Override @Override
public String getNextPageUrl() throws IOException, ExtractionException { public String getNextPageUrl() throws IOException, ExtractionException {
// initial page does not load any comments but is required to get session token // initial page does not load any comments but is required to get comments token
// and cookies
super.fetchPage(); super.fetchPage();
return initPage.getNextPageUrl(); return initPage.getNextPageUrl();
} }
@ -64,7 +61,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray arr; JsonArray arr;
try { try {
arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations"); arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
} catch (Exception e) { } catch (Exception e) {
return ""; return "";
} }
@ -85,9 +82,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
params.put("action_get_comments", "1"); params.put("action_get_comments", "1");
params.put("pbj", "1"); params.put("pbj", "1");
params.put("ctoken", continuation); params.put("ctoken", continuation);
params.put("continuation", continuation);
try { try {
return "https://www.youtube.com/comment_service_ajax?" + getDataString(params); return "https://m.youtube.com/watch_comment?" + getDataString(params);
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not get next page url", e); throw new ParsingException("Could not get next page url", e);
} }
@ -101,8 +97,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
String ajaxResponse = makeAjaxRequest(pageUrl); String ajaxResponse = makeAjaxRequest(pageUrl);
JsonObject ajaxJson; JsonObject ajaxJson;
try { try {
ajaxJson = JsonParser.object().from(ajaxResponse); ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
} catch (JsonParserException e) { } catch (Exception e) {
throw new ParsingException("Could not parse json data for comments", e); throw new ParsingException("Could not parse json data for comments", e);
} }
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
@ -114,7 +110,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray contents; JsonArray contents;
try { try {
contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents"); contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
}catch(Exception e) { }catch(Exception e) {
//no comments //no comments
return; return;
@ -138,7 +134,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private void fetchTitle(JsonArray contents) { private void fetchTitle(JsonArray contents) {
if(null == title) { if(null == title) {
try { try {
title = (String) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText"); title = getYoutubeText((JsonObject) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
} catch (Exception e) { } catch (Exception e) {
title = "Youtube Comments"; title = "Youtube Comments";
} }
@ -152,11 +148,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
DownloadRequest request = new DownloadRequest(null, requestHeaders); DownloadRequest request = new DownloadRequest(null, requestHeaders);
DownloadResponse response = downloader.get(getUrl(), request); DownloadResponse response = downloader.get(getUrl(), request);
String responseBody = response.getResponseBody(); String responseBody = response.getResponseBody();
cookies = response.getResponseCookies();
sessionToken = findValue(responseBody, "XSRF_TOKEN\":\"", "\"");
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\""); ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_NAME\":", ","); ytClientName = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_NAME\":", ",");
String commentsTokenInside = findValue(responseBody, "itemSectionRenderer", "comment-item-section"); String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
initPage = getPage(getNextPageUrl(commentsToken)); initPage = getPage(getNextPageUrl(commentsToken));
} }
@ -168,20 +162,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException { private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
Map<String, String> postDataMap = new HashMap<>();
postDataMap.put("session_token", sessionToken);
String postData = getDataString(postDataMap);
Map<String, List<String>> requestHeaders = new HashMap<>(); Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Content-Type", Arrays.asList("application/x-www-form-urlencoded"));
requestHeaders.put("Accept", Arrays.asList("*/*")); requestHeaders.put("Accept", Arrays.asList("*/*"));
requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT)); requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
requestHeaders.put("X-YouTube-Client-Version", Arrays.asList(ytClientVersion)); requestHeaders.put("X-YouTube-Client-Version", Arrays.asList(ytClientVersion));
requestHeaders.put("X-YouTube-Client-Name", Arrays.asList(ytClientName)); requestHeaders.put("X-YouTube-Client-Name", Arrays.asList(ytClientName));
DownloadRequest request = new DownloadRequest(postData, requestHeaders); DownloadRequest request = new DownloadRequest(null, requestHeaders);
request.setRequestCookies(cookies);
return NewPipe.getDownloader().post(siteUrl, request).getResponseBody(); return NewPipe.getDownloader().get(siteUrl, request).getResponseBody();
} }
private String getDataString(Map<String, String> params) throws UnsupportedEncodingException { private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {
@ -205,4 +193,21 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
return doc.substring(beginIndex, endIndex); return doc.substring(beginIndex, endIndex);
} }
public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException {
try {
return (String) JsonUtils.getValue(object, "simpleText");
} catch (Exception e1) {
try {
JsonArray arr = (JsonArray) JsonUtils.getValue(object, "runs");
String result = "";
for(int i=0; i<arr.size();i++) {
result = result + (String) JsonUtils.getValue(arr.getObject(i), "text");
}
return result;
} catch (Exception e2) {
throw new ParsingException("Could not get text", e2);
}
}
}
} }

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View file

@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json; private final JsonObject json;
@ -35,7 +36,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return (String) JsonUtils.getValue(json, "authorText.simpleText"); return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author name", e); throw new ParsingException("Could not get author name", e);
} }
@ -44,8 +45,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getPublishedTime() throws ParsingException { public String getPublishedTime() throws ParsingException {
try { try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "publishedTimeText.runs"); return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "publishedTimeText"));
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e); throw new ParsingException("Could not get publishedTimeText", e);
} }
@ -63,14 +63,9 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getCommentText() throws ParsingException { public String getCommentText() throws ParsingException {
try { try {
return (String) JsonUtils.getValue(json, "contentText.simpleText"); return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "contentText"));
} catch (Exception e1) { } catch (Exception e) {
try { throw new ParsingException("Could not get comment text", e);
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "contentText.runs");
return (String) JsonUtils.getValue(arr.getObject(0), "text");
} catch (Exception e2) {
throw new ParsingException("Could not get comment text", e2);
}
} }
} }
@ -96,7 +91,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getAuthorName() throws ParsingException { public String getAuthorName() throws ParsingException {
try { try {
return (String) JsonUtils.getValue(json, "authorText.simpleText"); return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author name", e); throw new ParsingException("Could not get author name", e);
} }

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View file

@ -20,7 +20,7 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public String getUrl(String id) { public String getUrl(String id) {
return "https://www.youtube.com/watch?v=" + id; return "https://m.youtube.com/watch?v=" + id;
} }
@Override @Override
@ -102,6 +102,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
@Override @Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException { public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
return "https://www.youtube.com/watch?v=" + id; return getUrl(id);
} }
} }