CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

working

This commit is contained in:
CypherpunkSamurai 2023-03-10 14:51:13 +05:30
parent 65a31fd37c
commit 5ab267a97b
6 changed files with 137 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/XhService.java
View File

@ -18,6 +18,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.xh.extractors.XhSearchExtractor;
import org.schabi.newpipe.extractor.services.xh.extractors.XhStreamExtractor;
import org.schabi.newpipe.extractor.services.xh.linkHandler.XhSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.xh.linkHandler.XhStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
@ -28,7 +30,7 @@ public class XhService extends StreamingService {
final static String NAME = "Xhamsters";
// host
public final static String HOST = "xhamster18.desi";
public final static String HOST = "xhamster19.desi";
public final static String BASE_URL = "https://" + HOST + "/";
// constructor
@ -52,7 +54,7 @@ public class XhService extends StreamingService {
@Override
public LinkHandlerFactory getStreamLHFactory() {
return null;
return new XhStreamLinkHandlerFactory();
}
@Override
@ -67,7 +69,7 @@ public class XhService extends StreamingService {
@Override
public SearchQueryHandlerFactory getSearchQHFactory() {
return null;
return XhSearchQueryHandlerFactory.getInstance();
}
@Override

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchExtractor.java
View File

@ -11,7 +11,6 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -83,18 +82,19 @@ public class XhSearchExtractor extends SearchExtractor {
// Parse
String info_cdn = js.substring(0, js.length() - 1).replace("window.initials=", "");
System.out.println(info_cdn);
try {
// parse
final JsonObject info = JsonParser.object().from(info_cdn);
// list
final JsonArray list = info.getObject("SearchResult").getArray("models");
final JsonArray list = info.getObject("searchResult").getArray("models");
for (int i = 0; i < list.size(); i++) {
final JsonObject v = list.getObject(i);
collector.commit(new XhSearchStreamInfoExtractor(
collector.commit(new XhStreamInfoItemExtractor(
new XhVideo(
v.getInt("id"),
v.getString("title"),
@ -110,17 +110,27 @@ public class XhSearchExtractor extends SearchExtractor {
new HashMap<String, String>()
)
));
// log
System.out.println(v.getString("title"));
}
int currentPage = Integer.parseInt(page.getUrl().substring(page.getUrl().length() - 2));
assert list.size() < 46;
// split url
int l = url.split("/").length;
int currentPage = Integer.parseInt(
page.getUrl().split("/")[l - 1]
);
String nextUrl = null;
// if there is next page
String nextPageUrl = "";
if (currentPage < list.size()) {
nextUrl = page.getUrl().substring(0, page.getUrl().length() -2) + String.valueOf(currentPage + 1);
for(int i=0; i < l-1; i++){
nextPageUrl = nextPageUrl + page.getUrl().split("/")[i] + "/";
}
nextPageUrl = nextPageUrl + String.valueOf(currentPage + 1);
}
return new InfoItemsPage<>(collector, new Page(nextUrl));
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
} catch (JsonParserException ex) {
Logger.getLogger(XhSearchExtractor.class.getName()).log(Level.SEVERE, null, ex);
}

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhSearchStreamInfoExtractor.java
View File

@ -1,20 +0,0 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package org.schabi.newpipe.extractor.services.xh.extractors;
import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo;
/**
*
* @author Rakesh
*/
public class XhSearchStreamInfoExtractor extends XhStreamInfoItemExtractor {
public XhSearchStreamInfoExtractor(XhVideo result_item) {
super(result_item);
}
}

151
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/XhStreamExtractor.java
View File

@ -5,14 +5,17 @@
*/
package org.schabi.newpipe.extractor.services.xh.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -27,7 +30,11 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
// Parse
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.localization.Localization;
import static org.schabi.newpipe.extractor.services.xh.XhService.HOST;
import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
@ -39,60 +46,10 @@ public class XhStreamExtractor extends StreamExtractor {
// Object
XhVideo video;
public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
// get baseurl using predefined method getBaseUrl()
// perform regex to replace the url host
this.baseUrl = getBaseUrl();
// parse video info
try {
Document html = Jsoup.connect(this.baseUrl).get();
// Parse Content
String content = html.selectFirst("#initials-script").html();
String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", "");
// Use NanoJSON to convert to VideoItem
JsonObject json = JsonParser.object().from(info_cdn);
final JsonObject videoModel = json.getObject("videoModel");
// Video Info
final int id = videoModel.getInt("id");
final String title = videoModel.getString("title");
final String description = videoModel.getString("description");
final String thumbURL = videoModel.getString("thumbURL");
final int duration = videoModel.getInt("created");
final String pageURL = videoModel.getString("pageURL");
final int created = videoModel.getInt("created");
// Author
final String authorName = videoModel.getObject("author").getString("name");
final int authorID = videoModel.getObject("author").getInt("id");
final String authorThumb = videoModel.getObject("author").getString("thumbURL");
final String authorPageURL = videoModel.getObject("author").getString("pageURL");
HashMap<String, String> streams = new HashMap<>();
// iterate urls
final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
if (!mp4.isEmpty()) {
final Object keys[] = mp4.keySet().toArray();
for (int i=0; i < keys.length; i++) {
final String key = keys[i].toString();
streams.put(key, mp4.getString("key"));
}
}
// set the video object
this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams);
} catch (Exception e) {
throw new ParsingException(e.getMessage(), e);
}
public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public String getThumbnailUrl() throws ParsingException {
return this.video.getThumbURL();
@ -117,21 +74,35 @@ public class XhStreamExtractor extends StreamExtractor {
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
// video was not parsed / no video streams
if (this.video.getSources().isEmpty()) {
if (video.getSources().isEmpty()) {
return Collections.emptyList();
}
// new list
List<VideoStream> listStreams = new ArrayList<>();
final List<VideoStream> listStreams = new ArrayList<>();
// iterate hashmap for key values
for (Map.Entry<String, String> source: this.video.getSources().entrySet()) {
for (Map.Entry<String, String> source: video.getSources().entrySet()) {
System.out.println(source.getKey());
System.out.println(source.getValue());
MediaFormat format = MediaFormat.MPEG_4;
if (source.getValue().endsWith(".webm")) {
format = MediaFormat.WEBM;
}
DeliveryMethod deliver = DeliveryMethod.PROGRESSIVE_HTTP;
if (source.getValue().endsWith(".m3u8")) {
deliver = DeliveryMethod.HLS;
}
// add source to list
listStreams.add(
new VideoStream.Builder()
.setId(String.valueOf(this.video.getId()))
.setId(String.valueOf(video.getId()) + "-" + source.getKey())
.setContent(source.getValue(), true)
.setIsVideoOnly(false)
.setMediaFormat(source.getValue().endsWith(".webm")? MediaFormat.WEBM:MediaFormat.MPEG_4)
.setDeliveryMethod(deliver)
.setMediaFormat(format)
.setResolution(source.getKey())
.build()
);
@ -152,7 +123,71 @@ public class XhStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
// get baseurl using predefined method getBaseUrl()
// perform regex to replace the url host
String url = getLinkHandler().getUrl();
url = url.replaceAll("http?s\\:\\/\\/(xhamster?.+)\\/videos\\/", "https://" + HOST + "/videos/");
this.baseUrl = url;
final String html_ = downloader.get(url).responseBody();
// parse video info
try {
Document html = Jsoup.parse(html_);
// Parse Content
final Element e = html.getElementById("initials-script");
final String content = e.html();
String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", "");
// Use NanoJSON to convert to VideoItem
JsonObject json = JsonParser.object().from(info_cdn);
final JsonObject videoModel = json.getObject("videoModel");
// Video Info
final int id = videoModel.getInt("id");
final String title = videoModel.getString("title");
final String description = videoModel.getString("description");
final String thumbURL = videoModel.getString("thumbURL");
final int duration = videoModel.getInt("created");
final String pageURL = videoModel.getString("pageURL");
final int created = videoModel.getInt("created");
// Author
final String authorName = videoModel.getObject("author").getString("name");
final int authorID = videoModel.getObject("author").getInt("id");
final String authorThumb = videoModel.getObject("author").getString("thumbURL");
final String authorPageURL = videoModel.getObject("author").getString("pageURL");
HashMap<String, String> streams = new HashMap<>();
// iterate urls
JsonArray mp4 = json.getObject("xplayerSettings").getObject("sources").getObject("standard").getArray("mp4");
if (!mp4.isEmpty()) {
// headers
Map<String, List<String>> headers = new HashMap<>();
headers.put("Referer", Arrays.asList(baseUrl));
for (int i=0; i < mp4.size(); i++) {
JsonObject j = mp4.getObject(i);
String key = j.getString("quality");
String value = j.getString("url");
int responseCode = downloader.head(value, headers).responseCode();
System.out.println(responseCode);
streams.put(key, value);
}
}
// set the video object
this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams);
} catch (Exception e) {
throw new ExtractionException(e.getMessage(), e);
}
}
@Override

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/extractors/items/XhVideo.java
View File

@ -117,12 +117,12 @@ public class XhVideo {
HashMap<String, String> streams = new HashMap<>();
// iterate urls
final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
if (!mp4.isEmpty()) {
final Object keys[] = mp4.keySet().toArray();
for (int i = 0; i < keys.length; i++) {
final String key = keys[i].toString();
streams.put(key, mp4.getString("key"));
System.out.println(key);
}
}
@ -130,17 +130,17 @@ public class XhVideo {
}
public Map<String, String> getSources() {
if (sources.isEmpty() || sources == null) {
// check if pageurl is set
if (!pageURL.isBlank() || !pageURL.isEmpty()) {
// parse the page
try {
this.sources = getSourceURLs();
} catch (Exception e) {
e.printStackTrace();
}
}
}
// if (sources.isEmpty() || sources == null) {
// // check if pageurl is set
// if (!pageURL.isBlank() || !pageURL.isEmpty()) {
// // parse the page
// try {
// this.sources = getSourceURLs();
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
// }
return sources;
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/xh/linkHandler/XhSearchQueryHandlerFactory.java
View File

@ -8,6 +8,7 @@ package org.schabi.newpipe.extractor.services.xh.linkHandler;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import java.util.logging.Logger;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL;
@ -18,12 +19,19 @@ import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL;
*/
public class XhSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static XhSearchQueryHandlerFactory getInstance() {
return new XhSearchQueryHandlerFactory();
}
@Override
public String getUrl(final String query,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
try {
// log
System.out.println(BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1");
return BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1";
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("query \"" + query + "\" could not be encoded", e);