working
This commit is contained in:
CypherpunkSamurai
parent
65a31fd37c
commit
5ab267a97b
6 changed files with 137 additions and 102 deletions
|
|
@ -18,6 +18,8 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
|||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.xh.extractors.XhSearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.xh.extractors.XhStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.services.xh.linkHandler.XhSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.xh.linkHandler.XhStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
|
||||
|
|
@ -28,7 +30,7 @@ public class XhService extends StreamingService {
|
|||
final static String NAME = "Xhamsters";
|
||||
|
||||
// host
|
||||
public final static String HOST = "xhamster18.desi";
|
||||
public final static String HOST = "xhamster19.desi";
|
||||
public final static String BASE_URL = "https://" + HOST + "/";
|
||||
|
||||
// constructor
|
||||
|
|
@ -52,7 +54,7 @@ public class XhService extends StreamingService {
|
|||
|
||||
@Override
|
||||
public LinkHandlerFactory getStreamLHFactory() {
|
||||
return null;
|
||||
return new XhStreamLinkHandlerFactory();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -67,7 +69,7 @@ public class XhService extends StreamingService {
|
|||
|
||||
@Override
|
||||
public SearchQueryHandlerFactory getSearchQHFactory() {
|
||||
return null;
|
||||
return XhSearchQueryHandlerFactory.getInstance();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import com.grack.nanojson.JsonParser;
|
|||
import com.grack.nanojson.JsonParserException;
|
||||
import edu.umd.cs.findbugs.annotations.NonNull;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -83,18 +82,19 @@ public class XhSearchExtractor extends SearchExtractor {
|
|||
|
||||
// Parse
|
||||
String info_cdn = js.substring(0, js.length() - 1).replace("window.initials=", "");
|
||||
System.out.println(info_cdn);
|
||||
|
||||
try {
|
||||
// parse
|
||||
final JsonObject info = JsonParser.object().from(info_cdn);
|
||||
|
||||
// list
|
||||
final JsonArray list = info.getObject("SearchResult").getArray("models");
|
||||
final JsonArray list = info.getObject("searchResult").getArray("models");
|
||||
|
||||
for (int i = 0; i < list.size(); i++) {
|
||||
|
||||
final JsonObject v = list.getObject(i);
|
||||
collector.commit(new XhSearchStreamInfoExtractor(
|
||||
collector.commit(new XhStreamInfoItemExtractor(
|
||||
new XhVideo(
|
||||
v.getInt("id"),
|
||||
v.getString("title"),
|
||||
|
|
@ -110,17 +110,27 @@ public class XhSearchExtractor extends SearchExtractor {
|
|||
new HashMap<String, String>()
|
||||
)
|
||||
));
|
||||
// log
|
||||
System.out.println(v.getString("title"));
|
||||
}
|
||||
|
||||
int currentPage = Integer.parseInt(page.getUrl().substring(page.getUrl().length() - 2));
|
||||
assert list.size() < 46;
|
||||
// split url
|
||||
int l = url.split("/").length;
|
||||
|
||||
String nextUrl = null;
|
||||
int currentPage = Integer.parseInt(
|
||||
page.getUrl().split("/")[l - 1]
|
||||
);
|
||||
|
||||
// if there is next page
|
||||
String nextPageUrl = "";
|
||||
if (currentPage < list.size()) {
|
||||
nextUrl = page.getUrl().substring(0, page.getUrl().length() -2) + String.valueOf(currentPage + 1);
|
||||
for(int i=0; i < l-1; i++){
|
||||
nextPageUrl = nextPageUrl + page.getUrl().split("/")[i] + "/";
|
||||
}
|
||||
nextPageUrl = nextPageUrl + String.valueOf(currentPage + 1);
|
||||
}
|
||||
|
||||
return new InfoItemsPage<>(collector, new Page(nextUrl));
|
||||
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
|
||||
} catch (JsonParserException ex) {
|
||||
Logger.getLogger(XhSearchExtractor.class.getName()).log(Level.SEVERE, null, ex);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package org.schabi.newpipe.extractor.services.xh.extractors;
|
||||
|
||||
import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Rakesh
|
||||
*/
|
||||
public class XhSearchStreamInfoExtractor extends XhStreamInfoItemExtractor {
|
||||
|
||||
public XhSearchStreamInfoExtractor(XhVideo result_item) {
|
||||
super(result_item);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -5,14 +5,17 @@
|
|||
*/
|
||||
package org.schabi.newpipe.extractor.services.xh.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Logger;
|
||||
import org.schabi.newpipe.extractor.MediaFormat;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
|
|
@ -27,7 +30,11 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
|
|||
// Parse
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import static org.schabi.newpipe.extractor.services.xh.XhService.HOST;
|
||||
import org.schabi.newpipe.extractor.services.xh.extractors.items.XhVideo;
|
||||
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
|
||||
|
||||
|
||||
|
||||
|
|
@ -39,58 +46,8 @@ public class XhStreamExtractor extends StreamExtractor {
|
|||
// Object
|
||||
XhVideo video;
|
||||
|
||||
public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
|
||||
public XhStreamExtractor(StreamingService service, LinkHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
|
||||
// get baseurl using predefined method getBaseUrl()
|
||||
// perform regex to replace the url host
|
||||
this.baseUrl = getBaseUrl();
|
||||
|
||||
// parse video info
|
||||
try {
|
||||
Document html = Jsoup.connect(this.baseUrl).get();
|
||||
|
||||
// Parse Content
|
||||
String content = html.selectFirst("#initials-script").html();
|
||||
String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", "");
|
||||
|
||||
// Use NanoJSON to convert to VideoItem
|
||||
JsonObject json = JsonParser.object().from(info_cdn);
|
||||
final JsonObject videoModel = json.getObject("videoModel");
|
||||
|
||||
// Video Info
|
||||
final int id = videoModel.getInt("id");
|
||||
final String title = videoModel.getString("title");
|
||||
final String description = videoModel.getString("description");
|
||||
final String thumbURL = videoModel.getString("thumbURL");
|
||||
final int duration = videoModel.getInt("created");
|
||||
final String pageURL = videoModel.getString("pageURL");
|
||||
final int created = videoModel.getInt("created");
|
||||
// Author
|
||||
final String authorName = videoModel.getObject("author").getString("name");
|
||||
final int authorID = videoModel.getObject("author").getInt("id");
|
||||
final String authorThumb = videoModel.getObject("author").getString("thumbURL");
|
||||
final String authorPageURL = videoModel.getObject("author").getString("pageURL");
|
||||
|
||||
|
||||
HashMap<String, String> streams = new HashMap<>();
|
||||
|
||||
// iterate urls
|
||||
final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
|
||||
if (!mp4.isEmpty()) {
|
||||
final Object keys[] = mp4.keySet().toArray();
|
||||
for (int i=0; i < keys.length; i++) {
|
||||
final String key = keys[i].toString();
|
||||
streams.put(key, mp4.getString("key"));
|
||||
}
|
||||
}
|
||||
|
||||
// set the video object
|
||||
this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException(e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -117,21 +74,35 @@ public class XhStreamExtractor extends StreamExtractor {
|
|||
@Override
|
||||
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
|
||||
// video was not parsed / no video streams
|
||||
if (this.video.getSources().isEmpty()) {
|
||||
if (video.getSources().isEmpty()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// new list
|
||||
List<VideoStream> listStreams = new ArrayList<>();
|
||||
final List<VideoStream> listStreams = new ArrayList<>();
|
||||
// iterate hashmap for key values
|
||||
for (Map.Entry<String, String> source: this.video.getSources().entrySet()) {
|
||||
for (Map.Entry<String, String> source: video.getSources().entrySet()) {
|
||||
System.out.println(source.getKey());
|
||||
System.out.println(source.getValue());
|
||||
|
||||
MediaFormat format = MediaFormat.MPEG_4;
|
||||
if (source.getValue().endsWith(".webm")) {
|
||||
format = MediaFormat.WEBM;
|
||||
}
|
||||
|
||||
DeliveryMethod deliver = DeliveryMethod.PROGRESSIVE_HTTP;
|
||||
if (source.getValue().endsWith(".m3u8")) {
|
||||
deliver = DeliveryMethod.HLS;
|
||||
}
|
||||
|
||||
// add source to list
|
||||
listStreams.add(
|
||||
new VideoStream.Builder()
|
||||
.setId(String.valueOf(this.video.getId()))
|
||||
.setId(String.valueOf(video.getId()) + "-" + source.getKey())
|
||||
.setContent(source.getValue(), true)
|
||||
.setIsVideoOnly(false)
|
||||
.setMediaFormat(source.getValue().endsWith(".webm")? MediaFormat.WEBM:MediaFormat.MPEG_4)
|
||||
.setDeliveryMethod(deliver)
|
||||
.setMediaFormat(format)
|
||||
.setResolution(source.getKey())
|
||||
.build()
|
||||
);
|
||||
|
|
@ -152,7 +123,71 @@ public class XhStreamExtractor extends StreamExtractor {
|
|||
|
||||
@Override
|
||||
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
|
||||
// get baseurl using predefined method getBaseUrl()
|
||||
// perform regex to replace the url host
|
||||
String url = getLinkHandler().getUrl();
|
||||
url = url.replaceAll("http?s\\:\\/\\/(xhamster?.+)\\/videos\\/", "https://" + HOST + "/videos/");
|
||||
|
||||
this.baseUrl = url;
|
||||
|
||||
final String html_ = downloader.get(url).responseBody();
|
||||
|
||||
// parse video info
|
||||
try {
|
||||
Document html = Jsoup.parse(html_);
|
||||
|
||||
// Parse Content
|
||||
final Element e = html.getElementById("initials-script");
|
||||
final String content = e.html();
|
||||
|
||||
String info_cdn = content.substring(0, content.length() - 1).replace("window.initials=", "");
|
||||
|
||||
// Use NanoJSON to convert to VideoItem
|
||||
JsonObject json = JsonParser.object().from(info_cdn);
|
||||
final JsonObject videoModel = json.getObject("videoModel");
|
||||
|
||||
// Video Info
|
||||
final int id = videoModel.getInt("id");
|
||||
final String title = videoModel.getString("title");
|
||||
final String description = videoModel.getString("description");
|
||||
final String thumbURL = videoModel.getString("thumbURL");
|
||||
final int duration = videoModel.getInt("created");
|
||||
final String pageURL = videoModel.getString("pageURL");
|
||||
final int created = videoModel.getInt("created");
|
||||
// Author
|
||||
final String authorName = videoModel.getObject("author").getString("name");
|
||||
final int authorID = videoModel.getObject("author").getInt("id");
|
||||
final String authorThumb = videoModel.getObject("author").getString("thumbURL");
|
||||
final String authorPageURL = videoModel.getObject("author").getString("pageURL");
|
||||
|
||||
HashMap<String, String> streams = new HashMap<>();
|
||||
|
||||
// iterate urls
|
||||
JsonArray mp4 = json.getObject("xplayerSettings").getObject("sources").getObject("standard").getArray("mp4");
|
||||
if (!mp4.isEmpty()) {
|
||||
|
||||
// headers
|
||||
Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("Referer", Arrays.asList(baseUrl));
|
||||
|
||||
for (int i=0; i < mp4.size(); i++) {
|
||||
JsonObject j = mp4.getObject(i);
|
||||
|
||||
String key = j.getString("quality");
|
||||
String value = j.getString("url");
|
||||
|
||||
int responseCode = downloader.head(value, headers).responseCode();
|
||||
System.out.println(responseCode);
|
||||
|
||||
streams.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
// set the video object
|
||||
this.video = new XhVideo(id, title, description, duration, created, pageURL, thumbURL, authorID, authorName, authorThumb, authorPageURL, streams);
|
||||
} catch (Exception e) {
|
||||
throw new ExtractionException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -117,12 +117,12 @@ public class XhVideo {
|
|||
HashMap<String, String> streams = new HashMap<>();
|
||||
|
||||
// iterate urls
|
||||
final JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
|
||||
JsonObject mp4 = videoModel.getObject("sources").getObject("mp4");
|
||||
if (!mp4.isEmpty()) {
|
||||
final Object keys[] = mp4.keySet().toArray();
|
||||
for (int i = 0; i < keys.length; i++) {
|
||||
final String key = keys[i].toString();
|
||||
streams.put(key, mp4.getString("key"));
|
||||
System.out.println(key);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,17 +130,17 @@ public class XhVideo {
|
|||
}
|
||||
|
||||
public Map<String, String> getSources() {
|
||||
if (sources.isEmpty() || sources == null) {
|
||||
// check if pageurl is set
|
||||
if (!pageURL.isBlank() || !pageURL.isEmpty()) {
|
||||
// parse the page
|
||||
try {
|
||||
this.sources = getSourceURLs();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
// if (sources.isEmpty() || sources == null) {
|
||||
// // check if pageurl is set
|
||||
// if (!pageURL.isBlank() || !pageURL.isEmpty()) {
|
||||
// // parse the page
|
||||
// try {
|
||||
// this.sources = getSourceURLs();
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
return sources;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ package org.schabi.newpipe.extractor.services.xh.linkHandler;
|
|||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
import java.util.logging.Logger;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
|
||||
import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL;
|
||||
|
|
@ -18,12 +19,19 @@ import static org.schabi.newpipe.extractor.services.xh.XhService.BASE_URL;
|
|||
*/
|
||||
public class XhSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
|
||||
public static XhSearchQueryHandlerFactory getInstance() {
|
||||
return new XhSearchQueryHandlerFactory();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl(final String query,
|
||||
final List<String> contentFilter,
|
||||
final String sortFilter) throws ParsingException {
|
||||
try {
|
||||
|
||||
// log
|
||||
System.out.println(BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1");
|
||||
|
||||
return BASE_URL + "/search/" + URLEncoder.encode(query, "UTF-8") + "/1";
|
||||
} catch (final UnsupportedEncodingException e) {
|
||||
throw new ParsingException("query \"" + query + "\" could not be encoded", e);
|
||||
|
|
|
|||
Loading…
Reference in a new issue