From 5ab267a97bce9c7bcfd51beea1263f89b6ddc619 Mon Sep 17 00:00:00 2001 From: CypherpunkSamurai Date: Fri, 10 Mar 2023 14:51:13 +0530 Subject: [PATCH] working --- .../extractor/services/xh/XhService.java | 8 +- .../xh/extractors/XhSearchExtractor.java | 26 ++- .../XhSearchStreamInfoExtractor.java | 20 --- .../xh/extractors/XhStreamExtractor.java | 151 +++++++++++------- .../services/xh/extractors/items/XhVideo.java | 26 +-- .../XhSearchQueryHandlerFactory.java | 8 + 6 files changed, 137 insertions(+), 102 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchStreamInfoExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/XhService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/XhService.java index 4337a9d8..d6ee63ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/XhService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/XhService.java @@ -18,6 +18,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.xh.extractors.XhSearchExtractor; import org.schabi.newpipe.extractor.services.xh.extractors.XhStreamExtractor; +import org.schabi.newpipe.extractor.services.xh.linkHandler.XhSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.xh.linkHandler.XhStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; @@ -28,7 +30,7 @@ public class XhService extends StreamingService { final static String NAME = "Xhamsters"; // host - public final static String HOST = "xhamster18.desi"; + public final static String HOST = "xhamster19.desi"; public final static String BASE_URL = "https://" + HOST + "/"; // constructor @@ -52,7 +54,7 @@ public class XhService extends StreamingService { @Override public LinkHandlerFactory getStreamLHFactory() { - return null; + return new XhStreamLinkHandlerFactory(); } @Override @@ -67,7 +69,7 @@ public class XhService extends StreamingService { @Override public SearchQueryHandlerFactory getSearchQHFactory() { - return null; + return XhSearchQueryHandlerFactory.getInstance(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchExtractor.java index 2132a386..b4d0fe81 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchExtractor.java @@ -11,7 +11,6 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -83,18 +82,19 @@ public class XhSearchExtractor extends SearchExtractor { // Parse String info_cdn = js.substring(0, js.length() - 1).replace("window.initials=", ""); + System.out.println(info_cdn); try { // parse final JsonObject info = JsonParser.object().from(info_cdn); // list - final JsonArray list = info.getObject("SearchResult").getArray("models"); + final JsonArray list = info.getObject("searchResult").getArray("models"); for (int i = 0; i < list.size(); i++) { final JsonObject v = list.getObject(i); - collector.commit(new XhSearchStreamInfoExtractor( + collector.commit(new XhStreamInfoItemExtractor( new XhVideo( v.getInt("id"), v.getString("title"), @@ -110,17 +110,27 @@ public class XhSearchExtractor extends SearchExtractor { new HashMap() ) )); + // log + System.out.println(v.getString("title")); } - int currentPage = Integer.parseInt(page.getUrl().substring(page.getUrl().length() - 2)); - assert list.size() < 46; + // split url + int l = url.split("/").length; + + int currentPage = Integer.parseInt( + page.getUrl().split("/")[l - 1] + ); - String nextUrl = null; + // if there is next page + String nextPageUrl = ""; if (currentPage < list.size()) { - nextUrl = page.getUrl().substring(0, page.getUrl().length() -2) + String.valueOf(currentPage + 1); + for(int i=0; i < l-1; i++){ + nextPageUrl = nextPageUrl + page.getUrl().split("/")[i] + "/"; + } + nextPageUrl = nextPageUrl + String.valueOf(currentPage + 1); } - return new InfoItemsPage<>(collector, new Page(nextUrl)); + return new InfoItemsPage<>(collector, new Page(nextPageUrl)); } catch (JsonParserException ex) { Logger.getLogger(XhSearchExtractor.class.getName()).log(Level.SEVERE, null, ex); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchStreamInfoExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchStreamInfoExtractor.java deleted file mode 100644 index 88de5027..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchStreamInfoExtractor.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package org.schabi.newpipe.extractor.services.xh.extractors; - -import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo; - -/** - * - * @author Rakesh - */ -public class XhSearchStreamInfoExtractor extends XhStreamInfoItemExtractor { - - public XhSearchStreamInfoExtractor(XhVideo result_item) { - super(result_item); - } - -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhStreamExtractor.java index 773af4f6..5645530f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhStreamExtractor.java @@ -5,14 +5,17 @@ */ package org.schabi.newpipe.extractor.services.xh.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Logger; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -27,7 +30,11 @@ import org.schabi.newpipe.extractor.stream.VideoStream; // Parse import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.localization.Localization; +import static org.schabi.newpipe.extractor.services.xh.XhService.HOST; import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; @@ -39,60 +46,10 @@ public class XhStreamExtractor extends StreamExtractor { // Object XhVideo video; - public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { - super(service, linkHandler); - - // get baseurl using predefined method getBaseUrl() - // perform regex to replace the url host - this.baseUrl = getBaseUrl(); - - // parse video info - try { - Document html = Jsoup.connect(this.baseUrl).get(); - - // Parse Content - String content = html.selectFirst("#initials-script").html(); - String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", ""); - - // Use NanoJSON to convert to VideoItem - JsonObject json = JsonParser.object().from(info_cdn); - final JsonObject videoModel = json.getObject("videoModel"); - - // Video Info - final int id = videoModel.getInt("id"); - final String title = videoModel.getString("title"); - final String description = videoModel.getString("description"); - final String thumbURL = videoModel.getString("thumbURL"); - final int duration = videoModel.getInt("created"); - final String pageURL = videoModel.getString("pageURL"); - final int created = videoModel.getInt("created"); - // Author - final String authorName = videoModel.getObject("author").getString("name"); - final int authorID = videoModel.getObject("author").getInt("id"); - final String authorThumb = videoModel.getObject("author").getString("thumbURL"); - final String authorPageURL = videoModel.getObject("author").getString("pageURL"); - - - HashMap streams = new HashMap<>(); - - // iterate urls - final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4"); - if (!mp4.isEmpty()) { - final Object keys[] = mp4.keySet().toArray(); - for (int i=0; i < keys.length; i++) { - final String key = keys[i].toString(); - streams.put(key, mp4.getString("key")); - } - } - - // set the video object - this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams); - } catch (Exception e) { - throw new ParsingException(e.getMessage(), e); - } - + public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) { + super(service, linkHandler); } - + @Override public String getThumbnailUrl() throws ParsingException { return this.video.getThumbURL(); @@ -117,21 +74,35 @@ public class XhStreamExtractor extends StreamExtractor { @Override public List getVideoStreams() throws IOException, ExtractionException { // video was not parsed / no video streams - if (this.video.getSources().isEmpty()) { + if (video.getSources().isEmpty()) { return Collections.emptyList(); } // new list - List listStreams = new ArrayList<>(); + final List listStreams = new ArrayList<>(); // iterate hashmap for key values - for (Map.Entry source: this.video.getSources().entrySet()) { + for (Map.Entry source: video.getSources().entrySet()) { + System.out.println(source.getKey()); + System.out.println(source.getValue()); + + MediaFormat format = MediaFormat.MPEG_4; + if (source.getValue().endsWith(".webm")) { + format = MediaFormat.WEBM; + } + + DeliveryMethod deliver = DeliveryMethod.PROGRESSIVE_HTTP; + if (source.getValue().endsWith(".m3u8")) { + deliver = DeliveryMethod.HLS; + } + // add source to list listStreams.add( new VideoStream.Builder() - .setId(String.valueOf(this.video.getId())) + .setId(String.valueOf(video.getId()) + "-" + source.getKey()) .setContent(source.getValue(), true) .setIsVideoOnly(false) - .setMediaFormat(source.getValue().endsWith(".webm")? MediaFormat.WEBM:MediaFormat.MPEG_4) + .setDeliveryMethod(deliver) + .setMediaFormat(format) .setResolution(source.getKey()) .build() ); @@ -152,7 +123,71 @@ public class XhStreamExtractor extends StreamExtractor { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { + // get baseurl using predefined method getBaseUrl() + // perform regex to replace the url host + String url = getLinkHandler().getUrl(); + url = url.replaceAll("http?s\\:\\/\\/(xhamster?.+)\\/videos\\/", "https://" + HOST + "/videos/"); + + this.baseUrl = url; + final String html_ = downloader.get(url).responseBody(); + + // parse video info + try { + Document html = Jsoup.parse(html_); + + // Parse Content + final Element e = html.getElementById("initials-script"); + final String content = e.html(); + + String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", ""); + + // Use NanoJSON to convert to VideoItem + JsonObject json = JsonParser.object().from(info_cdn); + final JsonObject videoModel = json.getObject("videoModel"); + + // Video Info + final int id = videoModel.getInt("id"); + final String title = videoModel.getString("title"); + final String description = videoModel.getString("description"); + final String thumbURL = videoModel.getString("thumbURL"); + final int duration = videoModel.getInt("created"); + final String pageURL = videoModel.getString("pageURL"); + final int created = videoModel.getInt("created"); + // Author + final String authorName = videoModel.getObject("author").getString("name"); + final int authorID = videoModel.getObject("author").getInt("id"); + final String authorThumb = videoModel.getObject("author").getString("thumbURL"); + final String authorPageURL = videoModel.getObject("author").getString("pageURL"); + + HashMap streams = new HashMap<>(); + + // iterate urls + JsonArray mp4 = json.getObject("xplayerSettings").getObject("sources").getObject("standard").getArray("mp4"); + if (!mp4.isEmpty()) { + + // headers + Map> headers = new HashMap<>(); + headers.put("Referer", Arrays.asList(baseUrl)); + + for (int i=0; i < mp4.size(); i++) { + JsonObject j = mp4.getObject(i); + + String key = j.getString("quality"); + String value = j.getString("url"); + + int responseCode = downloader.head(value, headers).responseCode(); + System.out.println(responseCode); + + streams.put(key, value); + } + } + + // set the video object + this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams); + } catch (Exception e) { + throw new ExtractionException(e.getMessage(), e); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/items/XhVideo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/items/XhVideo.java index cc1073be..be32bd7f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/items/XhVideo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/items/XhVideo.java @@ -117,12 +117,12 @@ public class XhVideo { HashMap streams = new HashMap<>(); // iterate urls - final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4"); + JsonObject mp4 = videoModel.getObject("sources").getObject("mp4"); if (!mp4.isEmpty()) { final Object keys[] = mp4.keySet().toArray(); for (int i = 0; i < keys.length; i++) { final String key = keys[i].toString(); - streams.put(key, mp4.getString("key")); + System.out.println(key); } } @@ -130,17 +130,17 @@ public class XhVideo { } public Map getSources() { - if (sources.isEmpty() || sources == null) { - // check if pageurl is set - if (!pageURL.isBlank() || !pageURL.isEmpty()) { - // parse the page - try { - this.sources = getSourceURLs(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } + // if (sources.isEmpty() || sources == null) { + // // check if pageurl is set + // if (!pageURL.isBlank() || !pageURL.isEmpty()) { + // // parse the page + // try { + // this.sources = getSourceURLs(); + // } catch (Exception e) { + // e.printStackTrace(); + // } + // } + // } return sources; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/linkHandler/XhSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/linkHandler/XhSearchQueryHandlerFactory.java index 56076962..3d562a49 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/linkHandler/XhSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/linkHandler/XhSearchQueryHandlerFactory.java @@ -8,6 +8,7 @@ package org.schabi.newpipe.extractor.services.xh.linkHandler; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.List; +import java.util.logging.Logger; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL; @@ -18,12 +19,19 @@ import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL; */ public class XhSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + public static XhSearchQueryHandlerFactory getInstance() { + return new XhSearchQueryHandlerFactory(); + } @Override public String getUrl(final String query, final List contentFilter, final String sortFilter) throws ParsingException { try { + + // log + System.out.println(BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1"); + return BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1"; } catch (final UnsupportedEncodingException e) { throw new ParsingException("query \"" + query + "\" could not be encoded", e);