Fix dash parser and more refactor
- Add new itags
This commit is contained in:
parent
bda65e83d6
commit
b1989c0a83
41 changed files with 802 additions and 751 deletions
|
@ -14,7 +14,7 @@ public abstract class Extractor implements Serializable {
|
||||||
this.urlIdHandler = urlIdHandler;
|
this.urlIdHandler = urlIdHandler;
|
||||||
this.serviceId = serviceId;
|
this.serviceId = serviceId;
|
||||||
this.url = url;
|
this.url = url;
|
||||||
this.previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId);
|
this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getUrl() {
|
public String getUrl() {
|
||||||
|
|
|
@ -11,9 +11,9 @@ public abstract class Info implements Serializable {
|
||||||
* Id of this Info object <br>
|
* Id of this Info object <br>
|
||||||
* e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7
|
* e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7
|
||||||
*/
|
*/
|
||||||
public String id = "";
|
public String id;
|
||||||
public String url = "";
|
public String url;
|
||||||
public String name = "";
|
public String name;
|
||||||
|
|
||||||
public List<Throwable> errors = new Vector<>();
|
public List<Throwable> errors = new Vector<>();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor;
|
package org.schabi.newpipe.extractor;
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Created by the-scrabi on 11.02.17.
|
* Created by the-scrabi on 11.02.17.
|
||||||
*
|
*
|
||||||
|
@ -22,14 +20,21 @@ import java.io.Serializable;
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public interface InfoItem extends Serializable {
|
import java.io.Serializable;
|
||||||
enum InfoType {
|
|
||||||
|
public abstract class InfoItem implements Serializable {
|
||||||
|
public enum InfoType {
|
||||||
STREAM,
|
STREAM,
|
||||||
PLAYLIST,
|
PLAYLIST,
|
||||||
CHANNEL
|
CHANNEL
|
||||||
}
|
}
|
||||||
|
|
||||||
InfoType infoType();
|
public InfoItem(InfoType infoType) {
|
||||||
String getTitle();
|
this.info_type = infoType;
|
||||||
String getLink();
|
}
|
||||||
|
|
||||||
|
public final InfoType info_type;
|
||||||
|
public int service_id = -1;
|
||||||
|
public String url;
|
||||||
|
public String name;
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ import java.util.Vector;
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class InfoItemCollector {
|
public abstract class InfoItemCollector {
|
||||||
private List<InfoItem> itemList = new Vector<>();
|
private List<InfoItem> itemList = new Vector<>();
|
||||||
private List<Throwable> errors = new Vector<>();
|
private List<Throwable> errors = new Vector<>();
|
||||||
private int serviceId = -1;
|
private int serviceId = -1;
|
||||||
|
|
|
@ -91,4 +91,17 @@ public enum MediaFormat {
|
||||||
}
|
}
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the MediaFormat with the supplied mime type
|
||||||
|
*
|
||||||
|
* @return MediaFormat associated with this mime type,
|
||||||
|
* or null if none match it.
|
||||||
|
*/
|
||||||
|
public static MediaFormat getFromMimeType(String mimeType) {
|
||||||
|
for (MediaFormat vf : MediaFormat.values()) {
|
||||||
|
if (vf.mimeType.equals(mimeType)) return vf;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor.channel;
|
package org.schabi.newpipe.extractor.channel;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.Extractor;
|
|
||||||
import org.schabi.newpipe.extractor.ListExtractor;
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
|
|
|
@ -29,21 +29,16 @@ import java.util.List;
|
||||||
|
|
||||||
public class ChannelInfo extends Info {
|
public class ChannelInfo extends Info {
|
||||||
|
|
||||||
public static ChannelInfo getInfo(ChannelExtractor extractor)
|
public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
|
||||||
throws ParsingException {
|
|
||||||
ChannelInfo info = new ChannelInfo();
|
ChannelInfo info = new ChannelInfo();
|
||||||
|
|
||||||
// important data
|
// important data
|
||||||
info.service_id = extractor.getServiceId();
|
info.service_id = extractor.getServiceId();
|
||||||
info.url = extractor.getUrl();
|
info.url = extractor.getUrl();
|
||||||
info.name = extractor.getChannelName();
|
|
||||||
info.hasMoreStreams = extractor.hasMoreStreams();
|
|
||||||
|
|
||||||
try {
|
|
||||||
info.id = extractor.getChannelId();
|
info.id = extractor.getChannelId();
|
||||||
} catch (Exception e) {
|
info.name = extractor.getChannelName();
|
||||||
info.errors.add(e);
|
info.has_more_streams = extractor.hasMoreStreams();
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
info.avatar_url = extractor.getAvatarUrl();
|
info.avatar_url = extractor.getAvatarUrl();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -75,10 +70,10 @@ public class ChannelInfo extends Info {
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String avatar_url = "";
|
public String avatar_url;
|
||||||
public String banner_url = "";
|
public String banner_url;
|
||||||
public String feed_url = "";
|
public String feed_url;
|
||||||
public List<InfoItem> related_streams = null;
|
public List<InfoItem> related_streams;
|
||||||
public long subscriber_count = -1;
|
public long subscriber_count = -1;
|
||||||
public boolean hasMoreStreams = false;
|
public boolean has_more_streams = false;
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,25 +22,14 @@ import org.schabi.newpipe.extractor.InfoItem;
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class ChannelInfoItem implements InfoItem {
|
public class ChannelInfoItem extends InfoItem {
|
||||||
|
|
||||||
public int serviceId = -1;
|
public String thumbnail_url;
|
||||||
public String channelName = "";
|
public String description;
|
||||||
public String thumbnailUrl = "";
|
public long subscriber_count = -1;
|
||||||
public String webPageUrl = "";
|
public long view_count = -1;
|
||||||
public String description = "";
|
|
||||||
public long subscriberCount = -1;
|
|
||||||
public long viewCount = -1;
|
|
||||||
|
|
||||||
public InfoType infoType() {
|
public ChannelInfoItem() {
|
||||||
return InfoType.CHANNEL;
|
super(InfoType.CHANNEL);
|
||||||
}
|
|
||||||
|
|
||||||
public String getTitle() {
|
|
||||||
return channelName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getLink() {
|
|
||||||
return webPageUrl;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,24 +31,24 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
|
||||||
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
|
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
|
||||||
ChannelInfoItem resultItem = new ChannelInfoItem();
|
ChannelInfoItem resultItem = new ChannelInfoItem();
|
||||||
// important information
|
// important information
|
||||||
resultItem.channelName = extractor.getChannelName();
|
resultItem.name = extractor.getChannelName();
|
||||||
|
|
||||||
resultItem.serviceId = getServiceId();
|
resultItem.service_id = getServiceId();
|
||||||
resultItem.webPageUrl = extractor.getWebPageUrl();
|
resultItem.url = extractor.getWebPageUrl();
|
||||||
|
|
||||||
// optional information
|
// optional information
|
||||||
try {
|
try {
|
||||||
resultItem.subscriberCount = extractor.getSubscriberCount();
|
resultItem.subscriber_count = extractor.getSubscriberCount();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
addError(e);
|
addError(e);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
resultItem.viewCount = extractor.getViewCount();
|
resultItem.view_count = extractor.getViewCount();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
addError(e);
|
addError(e);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
|
resultItem.thumbnail_url = extractor.getThumbnailUrl();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
addError(e);
|
addError(e);
|
||||||
}
|
}
|
||||||
|
|
11
exceptions/ContentNotAvailableException.java
Normal file
11
exceptions/ContentNotAvailableException.java
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
package org.schabi.newpipe.extractor.exceptions;
|
||||||
|
|
||||||
|
public class ContentNotAvailableException extends ParsingException {
|
||||||
|
public ContentNotAvailableException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ContentNotAvailableException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,6 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor.playlist;
|
package org.schabi.newpipe.extractor.playlist;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.Extractor;
|
|
||||||
import org.schabi.newpipe.extractor.ListExtractor;
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
|
|
|
@ -14,14 +14,10 @@ public class PlaylistInfo extends Info {
|
||||||
|
|
||||||
info.service_id = extractor.getServiceId();
|
info.service_id = extractor.getServiceId();
|
||||||
info.url = extractor.getUrl();
|
info.url = extractor.getUrl();
|
||||||
info.name = extractor.getPlaylistName();
|
|
||||||
info.hasMoreStreams = extractor.hasMoreStreams();
|
|
||||||
|
|
||||||
try {
|
|
||||||
info.id = extractor.getPlaylistId();
|
info.id = extractor.getPlaylistId();
|
||||||
} catch (Exception e) {
|
info.name = extractor.getPlaylistName();
|
||||||
info.errors.add(e);
|
info.has_more_streams = extractor.hasMoreStreams();
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
info.streams_count = extractor.getStreamsCount();
|
info.streams_count = extractor.getStreamsCount();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -63,12 +59,12 @@ public class PlaylistInfo extends Info {
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String avatar_url = "";
|
public String avatar_url;
|
||||||
public String banner_url = "";
|
public String banner_url;
|
||||||
public String uploader_url = "";
|
public String uploader_url;
|
||||||
public String uploader_name = "";
|
public String uploader_name;
|
||||||
public String uploader_avatar_url = "";
|
public String uploader_avatar_url;
|
||||||
public long streams_count = 0;
|
public long streams_count = 0;
|
||||||
public List<InfoItem> related_streams = null;
|
public List<InfoItem> related_streams;
|
||||||
public boolean hasMoreStreams = false;
|
public boolean has_more_streams;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,22 +2,15 @@ package org.schabi.newpipe.extractor.playlist;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItem;
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
|
|
||||||
public class PlaylistInfoItem implements InfoItem {
|
public class PlaylistInfoItem extends InfoItem {
|
||||||
|
|
||||||
public int serviceId = -1;
|
public String thumbnail_url;
|
||||||
public String name = "";
|
/**
|
||||||
public String thumbnailUrl = "";
|
* How many streams this playlist have
|
||||||
public String webPageUrl = "";
|
*/
|
||||||
|
public long streams_count = 0;
|
||||||
|
|
||||||
public InfoType infoType() {
|
public PlaylistInfoItem() {
|
||||||
return InfoType.PLAYLIST;
|
super(InfoType.PLAYLIST);
|
||||||
}
|
|
||||||
|
|
||||||
public String getTitle() {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getLink() {
|
|
||||||
return webPageUrl;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package org.schabi.newpipe.extractor.playlist;
|
package org.schabi.newpipe.extractor.playlist;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||||
|
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
public class PlaylistInfoItemCollector extends InfoItemCollector {
|
public class PlaylistInfoItemCollector extends InfoItemCollector {
|
||||||
|
@ -12,10 +13,16 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
|
||||||
final PlaylistInfoItem resultItem = new PlaylistInfoItem();
|
final PlaylistInfoItem resultItem = new PlaylistInfoItem();
|
||||||
|
|
||||||
resultItem.name = extractor.getPlaylistName();
|
resultItem.name = extractor.getPlaylistName();
|
||||||
resultItem.serviceId = getServiceId();
|
resultItem.service_id = getServiceId();
|
||||||
resultItem.webPageUrl = extractor.getWebPageUrl();
|
resultItem.url = extractor.getWebPageUrl();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
resultItem.thumbnailUrl = extractor.getThumbnailUrl();
|
resultItem.thumbnail_url = extractor.getThumbnailUrl();
|
||||||
|
} catch (Exception e) {
|
||||||
|
addError(e);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
resultItem.streams_count = extractor.getStreamsCount();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
addError(e);
|
addError(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,4 +6,5 @@ public interface PlaylistInfoItemExtractor {
|
||||||
String getThumbnailUrl() throws ParsingException;
|
String getThumbnailUrl() throws ParsingException;
|
||||||
String getPlaylistName() throws ParsingException;
|
String getPlaylistName() throws ParsingException;
|
||||||
String getWebPageUrl() throws ParsingException;
|
String getWebPageUrl() throws ParsingException;
|
||||||
|
long getStreamsCount() throws ParsingException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package org.schabi.newpipe.extractor.search;
|
package org.schabi.newpipe.extractor.search;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
|
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
|
@ -30,15 +29,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class InfoItemSearchCollector extends InfoItemCollector {
|
public class InfoItemSearchCollector extends InfoItemCollector {
|
||||||
private String suggestion = "";
|
private String suggestion;
|
||||||
private StreamInfoItemCollector streamCollector;
|
private StreamInfoItemCollector streamCollector;
|
||||||
private ChannelInfoItemCollector channelCollector;
|
private ChannelInfoItemCollector channelCollector;
|
||||||
|
|
||||||
SearchResult result = new SearchResult();
|
private SearchResult result = new SearchResult();
|
||||||
|
|
||||||
InfoItemSearchCollector(UrlIdHandler handler, int serviceId) {
|
InfoItemSearchCollector(int serviceId) {
|
||||||
super(serviceId);
|
super(serviceId);
|
||||||
streamCollector = new StreamInfoItemCollector(handler, serviceId);
|
streamCollector = new StreamInfoItemCollector(serviceId);
|
||||||
channelCollector = new ChannelInfoItemCollector(serviceId);
|
channelCollector = new ChannelInfoItemCollector(serviceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor.search;
|
package org.schabi.newpipe.extractor.search;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -28,7 +27,7 @@ import java.util.EnumSet;
|
||||||
|
|
||||||
public abstract class SearchEngine {
|
public abstract class SearchEngine {
|
||||||
public enum Filter {
|
public enum Filter {
|
||||||
STREAM, CHANNEL, PLAY_LIST
|
STREAM, CHANNEL, PLAYLIST
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class NothingFoundException extends ExtractionException {
|
public static class NothingFoundException extends ExtractionException {
|
||||||
|
@ -39,8 +38,8 @@ public abstract class SearchEngine {
|
||||||
|
|
||||||
private InfoItemSearchCollector collector;
|
private InfoItemSearchCollector collector;
|
||||||
|
|
||||||
public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
|
public SearchEngine(int serviceId) {
|
||||||
collector = new InfoItemSearchCollector(urlIdHandler, serviceId);
|
collector = new InfoItemSearchCollector(serviceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected InfoItemSearchCollector getInfoItemSearchCollector() {
|
protected InfoItemSearchCollector getInfoItemSearchCollector() {
|
||||||
|
|
|
@ -49,7 +49,7 @@ public class SearchResult {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String suggestion = "";
|
public String suggestion;
|
||||||
public List<InfoItem> resultList = new Vector<>();
|
public List<InfoItem> resultList = new Vector<>();
|
||||||
public List<Throwable> errors = new Vector<>();
|
public List<Throwable> errors = new Vector<>();
|
||||||
}
|
}
|
||||||
|
|
160
services/youtube/ItagItem.java
Normal file
160
services/youtube/ItagItem.java
Normal file
|
@ -0,0 +1,160 @@
|
||||||
|
package org.schabi.newpipe.extractor.services.youtube;
|
||||||
|
|
||||||
|
import org.schabi.newpipe.extractor.MediaFormat;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
|
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
|
||||||
|
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
|
||||||
|
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
|
||||||
|
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
|
||||||
|
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
|
||||||
|
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
|
||||||
|
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
|
||||||
|
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
|
||||||
|
|
||||||
|
public class ItagItem {
|
||||||
|
/**
|
||||||
|
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
|
||||||
|
*/
|
||||||
|
private static final ItagItem[] ITAG_LIST = {
|
||||||
|
/////////////////////////////////////////////////////
|
||||||
|
// VIDEO ID Type Format Resolution FPS ///
|
||||||
|
///////////////////////////////////////////////////
|
||||||
|
new ItagItem(17, VIDEO, v3GPP, "144p"),
|
||||||
|
new ItagItem(36, VIDEO, v3GPP, "240p"),
|
||||||
|
|
||||||
|
new ItagItem(18, VIDEO, MPEG_4, "360p"),
|
||||||
|
new ItagItem(34, VIDEO, MPEG_4, "360p"),
|
||||||
|
new ItagItem(35, VIDEO, MPEG_4, "480p"),
|
||||||
|
new ItagItem(59, VIDEO, MPEG_4, "480p"),
|
||||||
|
new ItagItem(78, VIDEO, MPEG_4, "480p"),
|
||||||
|
new ItagItem(22, VIDEO, MPEG_4, "720p"),
|
||||||
|
new ItagItem(37, VIDEO, MPEG_4, "1080p"),
|
||||||
|
new ItagItem(38, VIDEO, MPEG_4, "1080p"),
|
||||||
|
|
||||||
|
new ItagItem(43, VIDEO, WEBM, "360p"),
|
||||||
|
new ItagItem(44, VIDEO, WEBM, "480p"),
|
||||||
|
new ItagItem(45, VIDEO, WEBM, "720p"),
|
||||||
|
new ItagItem(46, VIDEO, WEBM, "1080p"),
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////
|
||||||
|
// AUDIO ID ItagType Format Bitrate ///
|
||||||
|
//////////////////////////////////////////////////////////////////
|
||||||
|
// Disable Opus codec as it's not well supported in older devices
|
||||||
|
// new ItagItem(249, AUDIO, WEBMA, 50),
|
||||||
|
// new ItagItem(250, AUDIO, WEBMA, 70),
|
||||||
|
// new ItagItem(251, AUDIO, WEBMA, 16),
|
||||||
|
new ItagItem(171, AUDIO, WEBMA, 128),
|
||||||
|
new ItagItem(172, AUDIO, WEBMA, 256),
|
||||||
|
new ItagItem(139, AUDIO, M4A, 48),
|
||||||
|
new ItagItem(140, AUDIO, M4A, 128),
|
||||||
|
new ItagItem(141, AUDIO, M4A, 256),
|
||||||
|
|
||||||
|
/// VIDEO ONLY ////////////////////////////////////////////
|
||||||
|
// ID Type Format Resolution FPS ///
|
||||||
|
/////////////////////////////////////////////////////////
|
||||||
|
// Don't add VideoOnly streams that have normal variants
|
||||||
|
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
|
||||||
|
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
|
||||||
|
// new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
|
||||||
|
new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
|
||||||
|
new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
|
||||||
|
// new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
|
||||||
|
new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
|
||||||
|
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
|
||||||
|
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
|
||||||
|
new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
|
||||||
|
|
||||||
|
new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
|
||||||
|
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
|
||||||
|
// new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
|
||||||
|
new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
|
||||||
|
new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
|
||||||
|
new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),
|
||||||
|
new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
|
||||||
|
new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
|
||||||
|
new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
|
||||||
|
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
|
||||||
|
new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
|
||||||
|
new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
|
||||||
|
new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
|
||||||
|
new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
|
||||||
|
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
|
||||||
|
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
|
||||||
|
};
|
||||||
|
|
||||||
|
/*//////////////////////////////////////////////////////////////////////////
|
||||||
|
// Utils
|
||||||
|
//////////////////////////////////////////////////////////////////////////*/
|
||||||
|
|
||||||
|
public static boolean isSupported(int itag) {
|
||||||
|
for (ItagItem item : ITAG_LIST) {
|
||||||
|
if (itag == item.id) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ItagItem getItag(int itagId) throws ParsingException {
|
||||||
|
for (ItagItem item : ITAG_LIST) {
|
||||||
|
if (itagId == item.id) {
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
/*//////////////////////////////////////////////////////////////////////////
|
||||||
|
// Contructors and misc
|
||||||
|
//////////////////////////////////////////////////////////////////////////*/
|
||||||
|
|
||||||
|
public enum ItagType {
|
||||||
|
AUDIO,
|
||||||
|
VIDEO,
|
||||||
|
VIDEO_ONLY
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
|
||||||
|
*/
|
||||||
|
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
|
||||||
|
this.id = id;
|
||||||
|
this.itagType = type;
|
||||||
|
this.mediaFormatId = format.id;
|
||||||
|
this.resolutionString = resolution;
|
||||||
|
this.fps = 30;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor for videos.
|
||||||
|
*
|
||||||
|
* @param resolution string that will be used in the frontend
|
||||||
|
*/
|
||||||
|
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
|
||||||
|
this.id = id;
|
||||||
|
this.itagType = type;
|
||||||
|
this.mediaFormatId = format.id;
|
||||||
|
this.resolutionString = resolution;
|
||||||
|
this.fps = fps;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
|
||||||
|
this.id = id;
|
||||||
|
this.itagType = type;
|
||||||
|
this.mediaFormatId = format.id;
|
||||||
|
this.avgBitrate = avgBitrate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int id;
|
||||||
|
public ItagType itagType;
|
||||||
|
public int mediaFormatId;
|
||||||
|
|
||||||
|
// Audio fields
|
||||||
|
public int avgBitrate = -1;
|
||||||
|
|
||||||
|
// Video fields
|
||||||
|
public String resolutionString;
|
||||||
|
public int fps = -1;
|
||||||
|
|
||||||
|
}
|
|
@ -13,10 +13,11 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||||
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -135,7 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
if (subscriberCount == -1) {
|
if (subscriberCount == -1) {
|
||||||
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
|
||||||
if (el != null) {
|
if (el != null) {
|
||||||
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+", ""));
|
subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
|
||||||
} else {
|
} else {
|
||||||
throw new ParsingException("Could not get subscriber count");
|
throw new ParsingException("Could not get subscriber count");
|
||||||
}
|
}
|
||||||
|
@ -164,7 +165,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
throw new ExtractionException("Channel doesn't have more streams");
|
throw new ExtractionException("Channel doesn't have more streams");
|
||||||
}
|
}
|
||||||
|
|
||||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
|
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||||
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
|
||||||
|
|
||||||
|
@ -223,8 +224,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
|
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
|
||||||
collector.commit(new StreamInfoItemExtractor() {
|
collector.commit(new StreamInfoItemExtractor() {
|
||||||
@Override
|
@Override
|
||||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
public StreamType getStreamType() throws ParsingException {
|
||||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
return StreamType.VIDEO_STREAM;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -302,7 +303,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
output = input.replaceAll("\\D+", "");
|
output = Utils.removeNonDigitCharacters(input);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return Long.parseLong(output);
|
return Long.parseLong(output);
|
||||||
|
|
|
@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
|
||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Created by Christian Schabesberger on 12.02.17.
|
* Created by Christian Schabesberger on 12.02.17.
|
||||||
|
@ -62,7 +63,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
||||||
if (subsEl == null) {
|
if (subsEl == null) {
|
||||||
return 0;
|
return 0;
|
||||||
} else {
|
} else {
|
||||||
return Long.parseLong(subsEl.text().replaceAll("\\D+", ""));
|
return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,7 +73,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
|
||||||
if (metaEl == null) {
|
if (metaEl == null) {
|
||||||
return 0;
|
return 0;
|
||||||
} else {
|
} else {
|
||||||
return Long.parseLong(metaEl.text().replaceAll("\\D+", ""));
|
return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,18 +26,29 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
|
||||||
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
|
public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
|
||||||
|
|
||||||
|
private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
|
||||||
|
private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
|
||||||
|
|
||||||
|
public static YoutubeChannelUrlIdHandler getInstance() {
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public String getUrl(String channelId) {
|
public String getUrl(String channelId) {
|
||||||
return "https://www.youtube.com/" + channelId;
|
return "https://www.youtube.com/" + channelId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public String getId(String siteUrl) throws ParsingException {
|
public String getId(String siteUrl) throws ParsingException {
|
||||||
return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl);
|
return Parser.matchGroup1(ID_PATTERN, siteUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public String cleanUrl(String siteUrl) throws ParsingException {
|
public String cleanUrl(String siteUrl) throws ParsingException {
|
||||||
return getUrl(getId(siteUrl));
|
return getUrl(getId(siteUrl));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public boolean acceptUrl(String videoUrl) {
|
public boolean acceptUrl(String videoUrl) {
|
||||||
return (videoUrl.contains("youtube") ||
|
return (videoUrl.contains("youtube") ||
|
||||||
videoUrl.contains("youtu.be")) &&
|
videoUrl.contains("youtu.be")) &&
|
||||||
|
|
|
@ -12,10 +12,11 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||||
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -157,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
streamsCount = Long.parseLong(input.replaceAll("\\D+", ""));
|
streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
|
||||||
} catch (NumberFormatException e) {
|
} catch (NumberFormatException e) {
|
||||||
// When there's no videos in a playlist, there's no number in the "innerHtml",
|
// When there's no videos in a playlist, there's no number in the "innerHtml",
|
||||||
// all characters that is not a number is removed, so we try to parse a empty string
|
// all characters that is not a number is removed, so we try to parse a empty string
|
||||||
|
@ -186,7 +187,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
throw new ExtractionException("Playlist doesn't have more streams");
|
throw new ExtractionException("Playlist doesn't have more streams");
|
||||||
}
|
}
|
||||||
|
|
||||||
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId());
|
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
|
||||||
setupNextStreamsAjax(NewPipe.getDownloader());
|
setupNextStreamsAjax(NewPipe.getDownloader());
|
||||||
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
|
||||||
|
|
||||||
|
@ -244,8 +245,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
|
||||||
for (final Element li : element.children()) {
|
for (final Element li : element.children()) {
|
||||||
collector.commit(new StreamInfoItemExtractor() {
|
collector.commit(new StreamInfoItemExtractor() {
|
||||||
@Override
|
@Override
|
||||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
public StreamType getStreamType() throws ParsingException {
|
||||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
return StreamType.VIDEO_STREAM;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -7,8 +7,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
|
||||||
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
|
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
|
||||||
|
|
||||||
|
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
|
||||||
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";
|
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";
|
||||||
|
|
||||||
|
public static YoutubePlaylistUrlIdHandler getInstance() {
|
||||||
|
return instance;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getUrl(String listId) {
|
public String getUrl(String listId) {
|
||||||
return "https://www.youtube.com/playlist?list=" + listId;
|
return "https://www.youtube.com/playlist?list=" + listId;
|
||||||
|
|
|
@ -5,7 +5,6 @@ import org.jsoup.nodes.Document;
|
||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
import org.schabi.newpipe.extractor.Downloader;
|
import org.schabi.newpipe.extractor.Downloader;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
|
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
|
||||||
import org.schabi.newpipe.extractor.search.SearchEngine;
|
import org.schabi.newpipe.extractor.search.SearchEngine;
|
||||||
|
@ -40,8 +39,8 @@ public class YoutubeSearchEngine extends SearchEngine {
|
||||||
private static final String TAG = YoutubeSearchEngine.class.toString();
|
private static final String TAG = YoutubeSearchEngine.class.toString();
|
||||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||||
|
|
||||||
public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) {
|
public YoutubeSearchEngine(int serviceId) {
|
||||||
super(urlIdHandler, serviceId);
|
super(serviceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -58,7 +58,7 @@ public class YoutubeService extends StreamingService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SearchEngine getSearchEngineInstance() {
|
public SearchEngine getSearchEngineInstance() {
|
||||||
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId());
|
return new YoutubeSearchEngine(getServiceId());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -68,13 +68,13 @@ public class YoutubeService extends StreamingService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public UrlIdHandler getChannelUrlIdHandlerInstance() {
|
public UrlIdHandler getChannelUrlIdHandlerInstance() {
|
||||||
return new YoutubeChannelUrlIdHandler();
|
return YoutubeChannelUrlIdHandler.getInstance();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
|
public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
|
||||||
return new YoutubePlaylistUrlIdHandler();
|
return YoutubePlaylistUrlIdHandler.getInstance();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -9,20 +9,21 @@ import org.mozilla.javascript.Context;
|
||||||
import org.mozilla.javascript.Function;
|
import org.mozilla.javascript.Function;
|
||||||
import org.mozilla.javascript.ScriptableObject;
|
import org.mozilla.javascript.ScriptableObject;
|
||||||
import org.schabi.newpipe.extractor.Downloader;
|
import org.schabi.newpipe.extractor.Downloader;
|
||||||
import org.schabi.newpipe.extractor.MediaFormat;
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
import org.schabi.newpipe.extractor.UrlIdHandler;
|
||||||
|
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
|
||||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||||
|
import org.schabi.newpipe.extractor.stream.Stream;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||||
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||||
import org.schabi.newpipe.extractor.utils.Parser;
|
import org.schabi.newpipe.extractor.utils.Parser;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -52,12 +53,11 @@ import java.util.regex.Pattern;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class YoutubeStreamExtractor extends StreamExtractor {
|
public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
|
||||||
public static final String HTTPS = "https:";
|
|
||||||
public static final String CONTENT = "content";
|
|
||||||
public static final String REGEX_INT = "[^\\d]";
|
|
||||||
|
|
||||||
// exceptions
|
/*//////////////////////////////////////////////////////////////////////////
|
||||||
|
// Exceptions
|
||||||
|
//////////////////////////////////////////////////////////////////////////*/
|
||||||
|
|
||||||
public class DecryptException extends ParsingException {
|
public class DecryptException extends ParsingException {
|
||||||
DecryptException(String message, Throwable cause) {
|
DecryptException(String message, Throwable cause) {
|
||||||
|
@ -65,8 +65,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// special content not available exceptions
|
|
||||||
|
|
||||||
public class GemaException extends ContentNotAvailableException {
|
public class GemaException extends ContentNotAvailableException {
|
||||||
GemaException(String message) {
|
GemaException(String message) {
|
||||||
super(message);
|
super(message);
|
||||||
|
@ -79,267 +77,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----------------
|
/*//////////////////////////////////////////////////////////////////////////*/
|
||||||
|
|
||||||
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
|
private Document doc;
|
||||||
// download the /get_video_info page. Since a certain date dashmpd url is only available over
|
private final String dirtyUrl;
|
||||||
// this /get_video_info page, so we always need to download this one to.
|
|
||||||
// %%video_id%% will be replaced by the actual video id
|
|
||||||
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
|
|
||||||
private static final String GET_VIDEO_INFO_URL =
|
|
||||||
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
|
|
||||||
// eltype is necessary for the url above
|
|
||||||
private static final String EL_INFO = "el=info";
|
|
||||||
|
|
||||||
public enum ItagType {
|
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
|
||||||
AUDIO,
|
super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
|
||||||
VIDEO,
|
dirtyUrl = pageUrl;
|
||||||
VIDEO_ONLY
|
fetchDocument();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class ItagItem {
|
/*//////////////////////////////////////////////////////////////////////////
|
||||||
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
|
// Impl
|
||||||
this.id = id;
|
//////////////////////////////////////////////////////////////////////////*/
|
||||||
this.itagType = type;
|
|
||||||
this.mediaFormatId = format.id;
|
|
||||||
this.resolutionString = res;
|
|
||||||
this.fps = fps;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
|
@Override
|
||||||
this(id, type, format, 0, samplingRate, bandWidth);
|
public String getId() throws ParsingException {
|
||||||
}
|
|
||||||
|
|
||||||
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) {
|
|
||||||
this.id = id;
|
|
||||||
this.itagType = type;
|
|
||||||
this.mediaFormatId = format.id;
|
|
||||||
this.avgBitrate = avgBitrate;
|
|
||||||
this.samplingRate = samplingRate;
|
|
||||||
this.bandWidth = bandWidth;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int id;
|
|
||||||
public ItagType itagType;
|
|
||||||
public int mediaFormatId;
|
|
||||||
public String resolutionString;
|
|
||||||
public int fps = -1;
|
|
||||||
public int avgBitrate = -1;
|
|
||||||
public int samplingRate = -1;
|
|
||||||
public int bandWidth = -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
|
|
||||||
*/
|
|
||||||
private static final ItagItem[] itagList = {
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// VIDEO ID ItagType Format Resolution FPS ///
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12),
|
|
||||||
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24),
|
|
||||||
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24),
|
|
||||||
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24),
|
|
||||||
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
|
|
||||||
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
|
|
||||||
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24),
|
|
||||||
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24),
|
|
||||||
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24),
|
|
||||||
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24),
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// AUDIO ID ItagType Format Bitrate SamplingR Bandwidth ///
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Disable Opus codec as it's not well supported in older devices
|
|
||||||
// new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0),
|
|
||||||
// new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0),
|
|
||||||
// new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0),
|
|
||||||
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0),
|
|
||||||
new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0),
|
|
||||||
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0),
|
|
||||||
new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0),
|
|
||||||
|
|
||||||
/// VIDEO ONLY ///////////////////////////////////////////////////////////////////
|
|
||||||
// ID ItagType Format Resolution FPS ///
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Don't add VideoOnly streams that have normal variants
|
|
||||||
// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24),
|
|
||||||
// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24),
|
|
||||||
// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24),
|
|
||||||
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30),
|
|
||||||
// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30),
|
|
||||||
new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60),
|
|
||||||
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30),
|
|
||||||
new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60),
|
|
||||||
new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30),
|
|
||||||
|
|
||||||
// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30),
|
|
||||||
new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
|
||||||
new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
|
||||||
new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
|
|
||||||
new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30),
|
|
||||||
new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30),
|
|
||||||
new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30),
|
|
||||||
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
|
|
||||||
new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
|
|
||||||
new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60),
|
|
||||||
new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60),
|
|
||||||
new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60),
|
|
||||||
new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
|
|
||||||
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
|
|
||||||
};
|
|
||||||
|
|
||||||
public static boolean itagIsSupported(int itag) {
|
|
||||||
for (ItagItem item : itagList) {
|
|
||||||
if (itag == item.id) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ItagItem getItagItem(int itag) throws ParsingException {
|
|
||||||
for (ItagItem item : itagList) {
|
|
||||||
if (itag == item.id) {
|
|
||||||
return item;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final String TAG = YoutubeStreamExtractor.class.toString();
|
|
||||||
private final Document doc;
|
|
||||||
private JSONObject playerArgs;
|
|
||||||
private boolean isAgeRestricted;
|
|
||||||
private Map<String, String> videoInfoPage;
|
|
||||||
|
|
||||||
// static values
|
|
||||||
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
|
||||||
|
|
||||||
// cached values
|
|
||||||
private static volatile String decryptionCode = "";
|
|
||||||
|
|
||||||
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
|
|
||||||
String pageUrl = "";
|
|
||||||
|
|
||||||
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
|
|
||||||
throws ExtractionException, IOException {
|
|
||||||
super(urlIdHandler, pageUrl, serviceId);
|
|
||||||
//most common videoInfo fields are now set in our superclass, for all services
|
|
||||||
this.pageUrl = pageUrl;
|
|
||||||
Downloader downloader = NewPipe.getDownloader();
|
|
||||||
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
|
|
||||||
doc = Jsoup.parse(pageContent, pageUrl);
|
|
||||||
JSONObject ytPlayerConfig;
|
|
||||||
String playerUrl;
|
|
||||||
|
|
||||||
// Check if the video is age restricted
|
|
||||||
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
|
||||||
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
|
|
||||||
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
|
|
||||||
String videoInfoPageString = downloader.download(videoInfoUrl);
|
|
||||||
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
|
|
||||||
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
|
|
||||||
isAgeRestricted = true;
|
|
||||||
} else {
|
|
||||||
ytPlayerConfig = getPlayerConfig(pageContent);
|
|
||||||
playerArgs = getPlayerArgs(ytPlayerConfig);
|
|
||||||
playerUrl = getPlayerUrl(ytPlayerConfig);
|
|
||||||
isAgeRestricted = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (decryptionCode.isEmpty()) {
|
|
||||||
decryptionCode = loadDecryptionCode(playerUrl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
|
|
||||||
try {
|
try {
|
||||||
String ytPlayerConfigRaw =
|
return getUrlIdHandler().getId(getUrl());
|
||||||
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
|
} catch (Exception e) {
|
||||||
return new JSONObject(ytPlayerConfigRaw);
|
throw new ParsingException("Could not get stream id");
|
||||||
} catch (Parser.RegexException e) {
|
|
||||||
String errorReason = getErrorMessage();
|
|
||||||
switch(errorReason) {
|
|
||||||
case "GEMA":
|
|
||||||
throw new GemaException(errorReason);
|
|
||||||
case "":
|
|
||||||
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
|
||||||
default:
|
|
||||||
throw new ContentNotAvailableException("Content not available", e);
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
throw new ParsingException("Could not parse yt player config", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
|
|
||||||
JSONObject playerArgs;
|
|
||||||
|
|
||||||
//attempt to load the youtube js player JSON arguments
|
|
||||||
boolean isLiveStream = false; //used to determine if this is a livestream or not
|
|
||||||
try {
|
|
||||||
playerArgs = playerConfig.getJSONObject("args");
|
|
||||||
|
|
||||||
// check if we have a live stream. We need to filter it, since its not yet supported.
|
|
||||||
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|
|
||||||
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
|
||||||
isLiveStream = true;
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
throw new ParsingException("Could not parse yt player config", e);
|
|
||||||
}
|
|
||||||
if (isLiveStream) {
|
|
||||||
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
|
|
||||||
}
|
|
||||||
|
|
||||||
return playerArgs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
|
|
||||||
try {
|
|
||||||
// The Youtube service needs to be initialized by downloading the
|
|
||||||
// js-Youtube-player. This is done in order to get the algorithm
|
|
||||||
// for decrypting cryptic signatures inside certain stream urls.
|
|
||||||
String playerUrl = "";
|
|
||||||
|
|
||||||
JSONObject ytAssets = playerConfig.getJSONObject("assets");
|
|
||||||
playerUrl = ytAssets.getString("js");
|
|
||||||
|
|
||||||
if (playerUrl.startsWith("//")) {
|
|
||||||
playerUrl = HTTPS + playerUrl;
|
|
||||||
}
|
|
||||||
return playerUrl;
|
|
||||||
} catch (JSONException e) {
|
|
||||||
throw new ParsingException(
|
|
||||||
"Could not load decryption code for the Youtube service.", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
|
|
||||||
try {
|
|
||||||
Downloader downloader = NewPipe.getDownloader();
|
|
||||||
String playerUrl = "";
|
|
||||||
String videoId = urlidhandler.getId(pageUrl);
|
|
||||||
String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
|
||||||
String embedPageContent = downloader.download(embedUrl);
|
|
||||||
//todo: find out if this can be reapaced by Parser.matchGroup1()
|
|
||||||
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
|
|
||||||
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
|
|
||||||
while (patternMatcher.find()) {
|
|
||||||
playerUrl = patternMatcher.group(1);
|
|
||||||
}
|
|
||||||
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
|
|
||||||
|
|
||||||
if (playerUrl.startsWith("//")) {
|
|
||||||
playerUrl = HTTPS + playerUrl;
|
|
||||||
}
|
|
||||||
return playerUrl;
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new ParsingException(
|
|
||||||
"Could load decryption code form restricted video for the Youtube service.", e);
|
|
||||||
} catch (ReCaptchaException e) {
|
|
||||||
throw new ReCaptchaException("reCaptcha Challenge requested");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -457,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
@Override
|
@Override
|
||||||
public String getDashMpdUrl() throws ParsingException {
|
public String getDashMpdUrl() throws ParsingException {
|
||||||
try {
|
try {
|
||||||
String dashManifestUrl = "";
|
String dashManifestUrl;
|
||||||
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
|
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
|
||||||
dashManifestUrl = videoInfoPage.get("dashmpd");
|
dashManifestUrl = videoInfoPage.get("dashmpd");
|
||||||
} else if (playerArgs.has("dashmpd")) {
|
} else if (playerArgs.has("dashmpd")) {
|
||||||
|
@ -479,7 +237,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<AudioStream> getAudioStreams() throws ParsingException {
|
public List<AudioStream> getAudioStreams() throws ParsingException {
|
||||||
Vector<AudioStream> audioStreams = new Vector<>();
|
Vector<AudioStream> audioStreams = new Vector<>();
|
||||||
|
@ -507,9 +264,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
|
|
||||||
int itag = Integer.parseInt(tags.get("itag"));
|
int itag = Integer.parseInt(tags.get("itag"));
|
||||||
|
|
||||||
if (itagIsSupported(itag)) {
|
if (ItagItem.isSupported(itag)) {
|
||||||
ItagItem itagItem = getItagItem(itag);
|
ItagItem itagItem = ItagItem.getItag(itag);
|
||||||
if (itagItem.itagType == ItagType.AUDIO) {
|
if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
|
||||||
String streamUrl = tags.get("url");
|
String streamUrl = tags.get("url");
|
||||||
// if video has a signature: decrypt it and add it to the url
|
// if video has a signature: decrypt it and add it to the url
|
||||||
if (tags.get("s") != null) {
|
if (tags.get("s") != null) {
|
||||||
|
@ -517,11 +274,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
audioStreams.add(new AudioStream(streamUrl,
|
AudioStream audioStream = new AudioStream(streamUrl, itagItem.mediaFormatId, itagItem.avgBitrate);
|
||||||
itagItem.mediaFormatId,
|
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
|
||||||
itagItem.avgBitrate,
|
audioStreams.add(audioStream);
|
||||||
itagItem.bandWidth,
|
}
|
||||||
itagItem.samplingRate));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -552,19 +308,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
|
|
||||||
int itag = Integer.parseInt(tags.get("itag"));
|
int itag = Integer.parseInt(tags.get("itag"));
|
||||||
|
|
||||||
if (itagIsSupported(itag)) {
|
if (ItagItem.isSupported(itag)) {
|
||||||
ItagItem itagItem = getItagItem(itag);
|
ItagItem itagItem = ItagItem.getItag(itag);
|
||||||
if (itagItem.itagType == ItagType.VIDEO) {
|
if (itagItem.itagType == ItagItem.ItagType.VIDEO) {
|
||||||
String streamUrl = tags.get("url");
|
String streamUrl = tags.get("url");
|
||||||
// if video has a signature: decrypt it and add it to the url
|
// if video has a signature: decrypt it and add it to the url
|
||||||
if (tags.get("s") != null) {
|
if (tags.get("s") != null) {
|
||||||
streamUrl = streamUrl + "&signature="
|
streamUrl = streamUrl + "&signature="
|
||||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||||
}
|
}
|
||||||
videoStreams.add(new VideoStream(
|
|
||||||
streamUrl,
|
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString);
|
||||||
itagItem.mediaFormatId,
|
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
|
||||||
itagItem.resolutionString));
|
videoStreams.add(videoStream);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -612,9 +369,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
|
|
||||||
int itag = Integer.parseInt(tags.get("itag"));
|
int itag = Integer.parseInt(tags.get("itag"));
|
||||||
|
|
||||||
if (itagIsSupported(itag)) {
|
if (ItagItem.isSupported(itag)) {
|
||||||
ItagItem itagItem = getItagItem(itag);
|
ItagItem itagItem = ItagItem.getItag(itag);
|
||||||
if (itagItem.itagType == ItagType.VIDEO_ONLY) {
|
if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
|
||||||
String streamUrl = tags.get("url");
|
String streamUrl = tags.get("url");
|
||||||
// if video has a signature: decrypt it and add it to the url
|
// if video has a signature: decrypt it and add it to the url
|
||||||
if (tags.get("s") != null) {
|
if (tags.get("s") != null) {
|
||||||
|
@ -622,11 +379,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
+ decryptSignature(tags.get("s"), decryptionCode);
|
+ decryptSignature(tags.get("s"), decryptionCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
videoOnlyStreams.add(new VideoStream(
|
VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString, true);
|
||||||
true, //isVideoOnly
|
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
|
||||||
streamUrl,
|
videoOnlyStreams.add(videoStream);
|
||||||
itagItem.mediaFormatId,
|
}
|
||||||
itagItem.resolutionString));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -649,7 +405,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
public int getTimeStamp() throws ParsingException {
|
public int getTimeStamp() throws ParsingException {
|
||||||
String timeStamp;
|
String timeStamp;
|
||||||
try {
|
try {
|
||||||
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
|
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
|
||||||
} catch (Parser.RegexException e) {
|
} catch (Parser.RegexException e) {
|
||||||
// catch this instantly since an url does not necessarily have to have a time stamp
|
// catch this instantly since an url does not necessarily have to have a time stamp
|
||||||
|
|
||||||
|
@ -730,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
|
//if this ckicks in our button has no content and thefore likes/dislikes are disabled
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
return Integer.parseInt(likesString.replaceAll(REGEX_INT, ""));
|
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
|
||||||
} catch (NumberFormatException nfe) {
|
} catch (NumberFormatException nfe) {
|
||||||
throw new ParsingException(
|
throw new ParsingException(
|
||||||
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
|
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
|
||||||
|
@ -750,7 +506,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
//if this kicks in our button has no content and therefore likes/dislikes are disabled
|
//if this kicks in our button has no content and therefore likes/dislikes are disabled
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, ""));
|
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
|
||||||
} catch (NumberFormatException nfe) {
|
} catch (NumberFormatException nfe) {
|
||||||
throw new ParsingException(
|
throw new ParsingException(
|
||||||
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
|
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
|
||||||
|
@ -788,11 +544,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getPageUrl() {
|
|
||||||
return pageUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getChannelUrl() throws ParsingException {
|
public String getChannelUrl() throws ParsingException {
|
||||||
try {
|
try {
|
||||||
|
@ -804,92 +555,186 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public StreamInfo.StreamType getStreamType() throws ParsingException {
|
public StreamType getStreamType() throws ParsingException {
|
||||||
//todo: if implementing livestream support this value should be generated dynamically
|
//todo: if implementing livestream support this value should be generated dynamically
|
||||||
return StreamInfo.StreamType.VIDEO_STREAM;
|
return StreamType.VIDEO_STREAM;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Provides information about links to other videos on the video page, such as related videos.
|
* {@inheritDoc}
|
||||||
* This is encapsulated in a StreamInfoItem object,
|
|
||||||
* which is a subset of the fields in a full StreamInfo.
|
|
||||||
*/
|
*/
|
||||||
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
|
|
||||||
return new StreamInfoItemExtractor() {
|
|
||||||
@Override
|
@Override
|
||||||
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException {
|
public String getErrorMessage() {
|
||||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
||||||
|
StringBuilder errorReason;
|
||||||
|
|
||||||
|
if (errorMessage == null || errorMessage.isEmpty()) {
|
||||||
|
errorReason = null;
|
||||||
|
} else if (errorMessage.contains("GEMA")) {
|
||||||
|
// Gema sometimes blocks youtube music content in germany:
|
||||||
|
// https://www.gema.de/en/
|
||||||
|
// Detailed description:
|
||||||
|
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
||||||
|
errorReason = new StringBuilder("GEMA");
|
||||||
|
} else {
|
||||||
|
errorReason = new StringBuilder(errorMessage);
|
||||||
|
errorReason.append(" ");
|
||||||
|
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
return errorReason != null ? errorReason.toString() : null;
|
||||||
public boolean isAd() throws ParsingException {
|
|
||||||
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
/*//////////////////////////////////////////////////////////////////////////
|
||||||
public String getWebPageUrl() throws ParsingException {
|
// Utils
|
||||||
return li.select("a.content-link").first().attr("abs:href");
|
//////////////////////////////////////////////////////////////////////////*/
|
||||||
|
|
||||||
|
private JSONObject playerArgs;
|
||||||
|
private boolean isAgeRestricted;
|
||||||
|
private Map<String, String> videoInfoPage;
|
||||||
|
|
||||||
|
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
|
||||||
|
private static final String HTTPS = "https:";
|
||||||
|
private static final String CONTENT = "content";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sometimes if the html page of youtube is already downloaded, youtube web page will internally
|
||||||
|
* download the /get_video_info page. Since a certain date dashmpd url is only available over
|
||||||
|
* this /get_video_info page, so we always need to download this one to.
|
||||||
|
* <p>
|
||||||
|
* %%video_id%% will be replaced by the actual video id
|
||||||
|
* $$el_type$$ will be replaced by the actual el_type (se the declarations below)
|
||||||
|
*/
|
||||||
|
private static final String GET_VIDEO_INFO_URL =
|
||||||
|
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
|
||||||
|
// eltype is necessary for the url above
|
||||||
|
private static final String EL_INFO = "el=info";
|
||||||
|
|
||||||
|
|
||||||
|
// static values
|
||||||
|
private static final String DECRYPTION_FUNC_NAME = "decrypt";
|
||||||
|
|
||||||
|
// cached values
|
||||||
|
private static volatile String decryptionCode = "";
|
||||||
|
|
||||||
|
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
|
||||||
|
Downloader downloader = NewPipe.getDownloader();
|
||||||
|
|
||||||
|
String pageContent = downloader.download(getUrl());
|
||||||
|
doc = Jsoup.parse(pageContent, getUrl());
|
||||||
|
|
||||||
|
JSONObject ytPlayerConfig;
|
||||||
|
String playerUrl;
|
||||||
|
|
||||||
|
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
|
||||||
|
String videoInfoPageString = downloader.download(videoInfoUrl);
|
||||||
|
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
|
||||||
|
|
||||||
|
// Check if the video is age restricted
|
||||||
|
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
|
||||||
|
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
|
||||||
|
isAgeRestricted = true;
|
||||||
|
} else {
|
||||||
|
ytPlayerConfig = getPlayerConfig(pageContent);
|
||||||
|
playerArgs = getPlayerArgs(ytPlayerConfig);
|
||||||
|
playerUrl = getPlayerUrl(ytPlayerConfig);
|
||||||
|
isAgeRestricted = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
if (decryptionCode.isEmpty()) {
|
||||||
public String getTitle() throws ParsingException {
|
decryptionCode = loadDecryptionCode(playerUrl);
|
||||||
//todo: check NullPointerException causing
|
}
|
||||||
return li.select("span.title").first().text();
|
|
||||||
//this page causes the NullPointerException, after finding it by searching for "tjvg":
|
|
||||||
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
|
||||||
public int getDuration() throws ParsingException {
|
|
||||||
return YoutubeParsingHelper.parseDurationString(
|
|
||||||
li.select("span.video-time").first().text());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getUploader() throws ParsingException {
|
|
||||||
return li.select("span.g-hovercard").first().text();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getUploadDate() throws ParsingException {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public long getViewCount() throws ParsingException {
|
|
||||||
//this line is unused
|
|
||||||
//String views = li.select("span.view-count").first().text();
|
|
||||||
|
|
||||||
//Log.i(TAG, "title:"+info.title);
|
|
||||||
//Log.i(TAG, "view count:"+views);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return Long.parseLong(li.select("span.view-count")
|
String ytPlayerConfigRaw =
|
||||||
.first().text().replaceAll(REGEX_INT, ""));
|
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
|
||||||
} catch (Exception e) {
|
return new JSONObject(ytPlayerConfigRaw);
|
||||||
//related videos sometimes have no view count
|
} catch (Parser.RegexException e) {
|
||||||
return 0;
|
String errorReason = getErrorMessage();
|
||||||
|
switch (errorReason) {
|
||||||
|
case "GEMA":
|
||||||
|
throw new GemaException(errorReason);
|
||||||
|
case "":
|
||||||
|
throw new ContentNotAvailableException("Content not available: player config empty", e);
|
||||||
|
default:
|
||||||
|
throw new ContentNotAvailableException("Content not available", e);
|
||||||
|
}
|
||||||
|
} catch (JSONException e) {
|
||||||
|
throw new ParsingException("Could not parse yt player config", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
|
||||||
public String getThumbnailUrl() throws ParsingException {
|
JSONObject playerArgs;
|
||||||
Element img = li.select("img").first();
|
|
||||||
String thumbnailUrl = img.attr("abs:src");
|
//attempt to load the youtube js player JSON arguments
|
||||||
// Sometimes youtube sends links to gif files which somehow seem to not exist
|
boolean isLiveStream = false; //used to determine if this is a livestream or not
|
||||||
// anymore. Items with such gif also offer a secondary image source. So we are going
|
try {
|
||||||
// to use that if we caught such an item.
|
playerArgs = playerConfig.getJSONObject("args");
|
||||||
if (thumbnailUrl.contains(".gif")) {
|
|
||||||
thumbnailUrl = img.attr("data-thumb");
|
// check if we have a live stream. We need to filter it, since its not yet supported.
|
||||||
|
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|
||||||
|
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
|
||||||
|
isLiveStream = true;
|
||||||
}
|
}
|
||||||
if (thumbnailUrl.startsWith("//")) {
|
} catch (JSONException e) {
|
||||||
thumbnailUrl = HTTPS + thumbnailUrl;
|
throw new ParsingException("Could not parse yt player config", e);
|
||||||
}
|
}
|
||||||
return thumbnailUrl;
|
if (isLiveStream) {
|
||||||
}
|
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return playerArgs;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
|
||||||
|
try {
|
||||||
|
// The Youtube service needs to be initialized by downloading the
|
||||||
|
// js-Youtube-player. This is done in order to get the algorithm
|
||||||
|
// for decrypting cryptic signatures inside certain stream urls.
|
||||||
|
String playerUrl;
|
||||||
|
|
||||||
|
JSONObject ytAssets = playerConfig.getJSONObject("assets");
|
||||||
|
playerUrl = ytAssets.getString("js");
|
||||||
|
|
||||||
|
if (playerUrl.startsWith("//")) {
|
||||||
|
playerUrl = HTTPS + playerUrl;
|
||||||
|
}
|
||||||
|
return playerUrl;
|
||||||
|
} catch (JSONException e) {
|
||||||
|
throw new ParsingException(
|
||||||
|
"Could not load decryption code for the Youtube service.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
|
||||||
|
try {
|
||||||
|
Downloader downloader = NewPipe.getDownloader();
|
||||||
|
String playerUrl = "";
|
||||||
|
String videoId = getUrlIdHandler().getId(pageUrl);
|
||||||
|
String embedUrl = "https://www.youtube.com/embed/" + videoId;
|
||||||
|
String embedPageContent = downloader.download(embedUrl);
|
||||||
|
//todo: find out if this can be reapaced by Parser.matchGroup1()
|
||||||
|
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
|
||||||
|
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
|
||||||
|
while (patternMatcher.find()) {
|
||||||
|
playerUrl = patternMatcher.group(1);
|
||||||
|
}
|
||||||
|
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
|
||||||
|
|
||||||
|
if (playerUrl.startsWith("//")) {
|
||||||
|
playerUrl = HTTPS + playerUrl;
|
||||||
|
}
|
||||||
|
return playerUrl;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ParsingException(
|
||||||
|
"Could load decryption code form restricted video for the Youtube service.", e);
|
||||||
|
} catch (ReCaptchaException e) {
|
||||||
|
throw new ReCaptchaException("reCaptcha Challenge requested");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String loadDecryptionCode(String playerUrl) throws DecryptException {
|
private String loadDecryptionCode(String playerUrl) throws DecryptException {
|
||||||
String decryptionFuncName;
|
String decryptionFuncName;
|
||||||
|
@ -935,8 +780,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
return decryptionCode;
|
return decryptionCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String decryptSignature(String encryptedSig, String decryptionCode)
|
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
|
||||||
throws DecryptException {
|
|
||||||
Context context = Context.enter();
|
Context context = Context.enter();
|
||||||
context.setOptimizationLevel(-1);
|
context.setOptimizationLevel(-1);
|
||||||
Object result = null;
|
Object result = null;
|
||||||
|
@ -953,28 +797,84 @@ public class YoutubeStreamExtractor extends StreamExtractor {
|
||||||
return result == null ? "" : result.toString();
|
return result == null ? "" : result.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* Provides information about links to other videos on the video page, such as related videos.
|
||||||
|
* This is encapsulated in a StreamInfoItem object,
|
||||||
|
* which is a subset of the fields in a full StreamInfo.
|
||||||
*/
|
*/
|
||||||
public String getErrorMessage() {
|
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
|
||||||
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
|
return new StreamInfoItemExtractor() {
|
||||||
StringBuilder errorReason;
|
@Override
|
||||||
|
public StreamType getStreamType() throws ParsingException {
|
||||||
if (errorMessage == null || errorMessage.isEmpty()) {
|
return StreamType.VIDEO_STREAM;
|
||||||
errorReason = null;
|
|
||||||
} else if(errorMessage.contains("GEMA")) {
|
|
||||||
// Gema sometimes blocks youtube music content in germany:
|
|
||||||
// https://www.gema.de/en/
|
|
||||||
// Detailed description:
|
|
||||||
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
|
|
||||||
errorReason = new StringBuilder("GEMA");
|
|
||||||
} else {
|
|
||||||
errorReason = new StringBuilder(errorMessage);
|
|
||||||
errorReason.append(" ");
|
|
||||||
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return errorReason != null ? errorReason.toString() : null;
|
@Override
|
||||||
|
public boolean isAd() throws ParsingException {
|
||||||
|
return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getWebPageUrl() throws ParsingException {
|
||||||
|
return li.select("a.content-link").first().attr("abs:href");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getTitle() throws ParsingException {
|
||||||
|
//todo: check NullPointerException causing
|
||||||
|
return li.select("span.title").first().text();
|
||||||
|
//this page causes the NullPointerException, after finding it by searching for "tjvg":
|
||||||
|
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getDuration() throws ParsingException {
|
||||||
|
return YoutubeParsingHelper.parseDurationString(
|
||||||
|
li.select("span.video-time").first().text());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getUploader() throws ParsingException {
|
||||||
|
return li.select("span.g-hovercard").first().text();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getUploadDate() throws ParsingException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getViewCount() throws ParsingException {
|
||||||
|
//this line is unused
|
||||||
|
//String views = li.select("span.view-count").first().text();
|
||||||
|
|
||||||
|
//Log.i(TAG, "title:"+info.title);
|
||||||
|
//Log.i(TAG, "view count:"+views);
|
||||||
|
|
||||||
|
try {
|
||||||
|
return Long.parseLong(Utils.removeNonDigitCharacters(
|
||||||
|
li.select("span.view-count").first().text()));
|
||||||
|
} catch (Exception e) {
|
||||||
|
//related videos sometimes have no view count
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getThumbnailUrl() throws ParsingException {
|
||||||
|
Element img = li.select("img").first();
|
||||||
|
String thumbnailUrl = img.attr("abs:src");
|
||||||
|
// Sometimes youtube sends links to gif files which somehow seem to not exist
|
||||||
|
// anymore. Items with such gif also offer a secondary image source. So we are going
|
||||||
|
// to use that if we caught such an item.
|
||||||
|
if (thumbnailUrl.contains(".gif")) {
|
||||||
|
thumbnailUrl = img.attr("data-thumb");
|
||||||
|
}
|
||||||
|
if (thumbnailUrl.startsWith("//")) {
|
||||||
|
thumbnailUrl = HTTPS + thumbnailUrl;
|
||||||
|
}
|
||||||
|
return thumbnailUrl;
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,8 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube;
|
||||||
import org.jsoup.nodes.Element;
|
import org.jsoup.nodes.Element;
|
||||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
|
||||||
|
import org.schabi.newpipe.extractor.stream.StreamType;
|
||||||
|
import org.schabi.newpipe.extractor.utils.Utils;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
||||||
|
@ -116,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output = input.replaceAll("[^0-9]+", "");
|
output = Utils.removeNonDigitCharacters(input);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return Long.parseLong(output);
|
return Long.parseLong(output);
|
||||||
|
@ -150,11 +151,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AbstractStreamInfo.StreamType getStreamType() {
|
public StreamType getStreamType() {
|
||||||
if (isLiveStream(item)) {
|
if (isLiveStream(item)) {
|
||||||
return AbstractStreamInfo.StreamType.LIVE_STREAM;
|
return StreamType.LIVE_STREAM;
|
||||||
} else {
|
} else {
|
||||||
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
|
return StreamType.VIDEO_STREAM;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -140,6 +140,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
|
||||||
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
|
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public String cleanUrl(String complexUrl) throws ParsingException {
|
public String cleanUrl(String complexUrl) throws ParsingException {
|
||||||
return getUrl(getId(complexUrl));
|
return getUrl(getId(complexUrl));
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,41 +0,0 @@
|
||||||
package org.schabi.newpipe.extractor.stream;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
|
|
||||||
* AbstractStreamInfo.java is part of NewPipe.
|
|
||||||
*
|
|
||||||
* NewPipe is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* NewPipe is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.Info;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Common properties between StreamInfo and StreamInfoItem.
|
|
||||||
*/
|
|
||||||
public abstract class AbstractStreamInfo extends Info {
|
|
||||||
public enum StreamType {
|
|
||||||
NONE, // placeholder to check if stream type was checked or not
|
|
||||||
VIDEO_STREAM,
|
|
||||||
AUDIO_STREAM,
|
|
||||||
LIVE_STREAM,
|
|
||||||
AUDIO_LIVE_STREAM,
|
|
||||||
FILE
|
|
||||||
}
|
|
||||||
|
|
||||||
public StreamType stream_type;
|
|
||||||
public String uploader = "";
|
|
||||||
public String thumbnail_url = "";
|
|
||||||
public String upload_date = "";
|
|
||||||
public long view_count = -1;
|
|
||||||
}
|
|
|
@ -1,7 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor.stream;
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Created by Christian Schabesberger on 04.03.16.
|
* Created by Christian Schabesberger on 04.03.16.
|
||||||
*
|
*
|
||||||
|
@ -22,31 +20,17 @@ import java.io.Serializable;
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class AudioStream implements Serializable {
|
public class AudioStream extends Stream {
|
||||||
public String url = "";
|
public int average_bitrate = -1;
|
||||||
public int format = -1;
|
|
||||||
public int bandwidth = -1;
|
|
||||||
public int sampling_rate = -1;
|
|
||||||
public int avgBitrate = -1;
|
|
||||||
|
|
||||||
public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) {
|
public AudioStream(String url, int format, int averageBitrate) {
|
||||||
this.url = url;
|
super(url, format);
|
||||||
this.format = format;
|
this.average_bitrate = averageBitrate;
|
||||||
this.avgBitrate = avgBitrate;
|
|
||||||
this.bandwidth = bandwidth;
|
|
||||||
this.sampling_rate = samplingRate;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// reveals whether two streams are the same, but have different urls
|
@Override
|
||||||
public boolean equalStats(AudioStream cmp) {
|
public boolean equalStats(Stream cmp) {
|
||||||
return format == cmp.format
|
return super.equalStats(cmp) && cmp instanceof AudioStream &&
|
||||||
&& bandwidth == cmp.bandwidth
|
average_bitrate == ((AudioStream) cmp).average_bitrate;
|
||||||
&& sampling_rate == cmp.sampling_rate
|
|
||||||
&& avgBitrate == cmp.avgBitrate;
|
|
||||||
}
|
|
||||||
|
|
||||||
// reveals whether two streams are equal
|
|
||||||
public boolean equals(AudioStream cmp) {
|
|
||||||
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
39
stream/Stream.java
Normal file
39
stream/Stream.java
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public abstract class Stream implements Serializable {
|
||||||
|
public String url;
|
||||||
|
public int format = -1;
|
||||||
|
|
||||||
|
public Stream(String url, int format) {
|
||||||
|
this.url = url;
|
||||||
|
this.format = format;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reveals whether two streams are the same, but have different urls
|
||||||
|
*/
|
||||||
|
public boolean equalStats(Stream cmp) {
|
||||||
|
return cmp != null && format == cmp.format;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reveals whether two Streams are equal
|
||||||
|
*/
|
||||||
|
public boolean equals(Stream cmp) {
|
||||||
|
return equalStats(cmp) && url.equals(cmp.url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the list already contains one stream with equals stats
|
||||||
|
*/
|
||||||
|
public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
|
||||||
|
if (stream == null || streamList == null) return false;
|
||||||
|
for (Stream cmpStream : streamList) {
|
||||||
|
if (stream.equalStats(cmpStream)) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
|
@ -31,20 +31,11 @@ import java.util.List;
|
||||||
*/
|
*/
|
||||||
public abstract class StreamExtractor extends Extractor {
|
public abstract class StreamExtractor extends Extractor {
|
||||||
|
|
||||||
public static class ContentNotAvailableException extends ParsingException {
|
|
||||||
public ContentNotAvailableException(String message) {
|
|
||||||
super(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public ContentNotAvailableException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
|
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
|
||||||
super(urlIdHandler, serviceId, url);
|
super(urlIdHandler, serviceId, url);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public abstract String getId() throws ParsingException;
|
||||||
public abstract int getTimeStamp() throws ParsingException;
|
public abstract int getTimeStamp() throws ParsingException;
|
||||||
public abstract String getTitle() throws ParsingException;
|
public abstract String getTitle() throws ParsingException;
|
||||||
public abstract String getDescription() throws ParsingException;
|
public abstract String getDescription() throws ParsingException;
|
||||||
|
@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
|
||||||
public abstract int getDislikeCount() throws ParsingException;
|
public abstract int getDislikeCount() throws ParsingException;
|
||||||
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
|
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
|
||||||
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
|
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
|
||||||
public abstract String getPageUrl();
|
public abstract StreamType getStreamType() throws ParsingException;
|
||||||
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Analyses the webpage's document and extracts any error message there might be.
|
* Analyses the webpage's document and extracts any error message there might be.
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
package org.schabi.newpipe.extractor.stream;
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
|
import org.schabi.newpipe.extractor.Info;
|
||||||
import org.schabi.newpipe.extractor.InfoItem;
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.utils.DashMpdParser;
|
import org.schabi.newpipe.extractor.utils.DashMpdParser;
|
||||||
|
|
||||||
|
@ -31,11 +32,11 @@ import java.util.Vector;
|
||||||
/**
|
/**
|
||||||
* Info object for opened videos, ie the video ready to play.
|
* Info object for opened videos, ie the video ready to play.
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("ALL")
|
@SuppressWarnings("WeakerAccess")
|
||||||
public class StreamInfo extends AbstractStreamInfo {
|
public class StreamInfo extends Info {
|
||||||
|
|
||||||
public static class StreamExctractException extends ExtractionException {
|
public static class StreamExtractException extends ExtractionException {
|
||||||
StreamExctractException(String message) {
|
StreamExtractException(String message) {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
public StreamInfo() {
|
public StreamInfo() {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a new StreamInfo object from an existing AbstractVideoInfo.
|
|
||||||
* All the shared properties are copied to the new StreamInfo.
|
|
||||||
*/
|
|
||||||
@SuppressWarnings("WeakerAccess")
|
|
||||||
public StreamInfo(AbstractStreamInfo avi) {
|
|
||||||
this.id = avi.id;
|
|
||||||
this.url = avi.url;
|
|
||||||
this.name = avi.name;
|
|
||||||
this.uploader = avi.uploader;
|
|
||||||
this.thumbnail_url = avi.thumbnail_url;
|
|
||||||
this.upload_date = avi.upload_date;
|
|
||||||
this.upload_date = avi.upload_date;
|
|
||||||
this.view_count = avi.view_count;
|
|
||||||
|
|
||||||
//todo: better than this
|
|
||||||
if (avi instanceof StreamInfoItem) {
|
|
||||||
//shitty String to convert code
|
|
||||||
/*
|
|
||||||
String dur = ((StreamInfoItem)avi).duration;
|
|
||||||
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
|
|
||||||
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
|
|
||||||
*/
|
|
||||||
this.duration = ((StreamInfoItem) avi).duration;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addException(Exception e) {
|
|
||||||
errors.add(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fills out the video info fields which are common to all services.
|
* Fills out the video info fields which are common to all services.
|
||||||
* Probably needs to be overridden by subclasses
|
* Probably needs to be overridden by subclasses
|
||||||
*/
|
*/
|
||||||
public static StreamInfo getVideoInfo(StreamExtractor extractor)
|
public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
|
||||||
throws ExtractionException, StreamExtractor.ContentNotAvailableException {
|
|
||||||
StreamInfo streamInfo = new StreamInfo();
|
StreamInfo streamInfo = new StreamInfo();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -95,7 +64,7 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
String errorMsg = extractor.getErrorMessage();
|
String errorMsg = extractor.getErrorMessage();
|
||||||
|
|
||||||
if (errorMsg != null) {
|
if (errorMsg != null) {
|
||||||
throw new StreamExtractor.ContentNotAvailableException(errorMsg);
|
throw new ContentNotAvailableException(errorMsg);
|
||||||
} else {
|
} else {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
return streamInfo;
|
return streamInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static StreamInfo extractImportantData(
|
private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
|
||||||
StreamInfo streamInfo, StreamExtractor extractor)
|
|
||||||
throws ExtractionException {
|
|
||||||
/* ---- important data, withoug the video can't be displayed goes here: ---- */
|
/* ---- important data, withoug the video can't be displayed goes here: ---- */
|
||||||
// if one of these is not available an exception is meant to be thrown directly into the frontend.
|
// if one of these is not available an exception is meant to be thrown directly into the frontend.
|
||||||
|
|
||||||
UrlIdHandler uiconv = extractor.getUrlIdHandler();
|
|
||||||
|
|
||||||
streamInfo.service_id = extractor.getServiceId();
|
streamInfo.service_id = extractor.getServiceId();
|
||||||
streamInfo.url = extractor.getPageUrl();
|
streamInfo.url = extractor.getUrl();
|
||||||
streamInfo.stream_type = extractor.getStreamType();
|
streamInfo.stream_type = extractor.getStreamType();
|
||||||
streamInfo.id = uiconv.getId(extractor.getPageUrl());
|
streamInfo.id = extractor.getId();
|
||||||
streamInfo.name = extractor.getTitle();
|
streamInfo.name = extractor.getTitle();
|
||||||
streamInfo.age_limit = extractor.getAgeLimit();
|
streamInfo.age_limit = extractor.getAgeLimit();
|
||||||
|
|
||||||
|
@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
return streamInfo;
|
return streamInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static StreamInfo extractStreams(
|
private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
|
||||||
StreamInfo streamInfo, StreamExtractor extractor)
|
|
||||||
throws ExtractionException {
|
|
||||||
/* ---- stream extraction goes here ---- */
|
/* ---- stream extraction goes here ---- */
|
||||||
// At least one type of stream has to be available,
|
// At least one type of stream has to be available,
|
||||||
// otherwise an exception will be thrown directly into the frontend.
|
// otherwise an exception will be thrown directly into the frontend.
|
||||||
|
@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
|
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
|
||||||
}
|
}
|
||||||
// also try to get streams from the dashMpd
|
|
||||||
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
|
|
||||||
if (streamInfo.audio_streams == null) {
|
|
||||||
streamInfo.audio_streams = new Vector<>();
|
|
||||||
}
|
|
||||||
//todo: make this quick and dirty solution a real fallback
|
|
||||||
// same as the quick and dirty above
|
|
||||||
try {
|
|
||||||
streamInfo.audio_streams.addAll(
|
|
||||||
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
|
|
||||||
} catch (Exception e) {
|
|
||||||
streamInfo.addException(
|
|
||||||
new ExtractionException("Couldn't get audio streams from dash mpd", e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/* Extract video stream url*/
|
/* Extract video stream url*/
|
||||||
try {
|
try {
|
||||||
streamInfo.video_streams = extractor.getVideoStreams();
|
streamInfo.video_streams = extractor.getVideoStreams();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
streamInfo.addException(
|
streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
|
||||||
new ExtractionException("Couldn't get video streams", e));
|
|
||||||
}
|
}
|
||||||
/* Extract video only stream url*/
|
/* Extract video only stream url*/
|
||||||
try {
|
try {
|
||||||
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
|
streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
streamInfo.addException(
|
streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
|
||||||
new ExtractionException("Couldn't get video only streams", e));
|
}
|
||||||
|
|
||||||
|
// Lists can be null if a exception was thrown during extraction
|
||||||
|
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
|
||||||
|
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
|
||||||
|
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
|
||||||
|
|
||||||
|
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
|
||||||
|
try {
|
||||||
|
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
|
||||||
|
// it has video, video only and audio streams and will only add to the list if it don't
|
||||||
|
// find a similar stream in the respective lists (calling Stream#equalStats).
|
||||||
|
DashMpdParser.getStreams(streamInfo);
|
||||||
|
} catch (Exception e) {
|
||||||
|
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
|
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
|
||||||
|
@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
|
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
|
||||||
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
|
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
|
||||||
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
|
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
|
||||||
throw new StreamExctractException(
|
throw new StreamExtractException(
|
||||||
"Could not get any stream. See error variable to get further details.");
|
"Could not get any stream. See error variable to get further details.");
|
||||||
}
|
}
|
||||||
|
|
||||||
return streamInfo;
|
return streamInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static StreamInfo extractOptionalData(
|
private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
|
||||||
StreamInfo streamInfo, StreamExtractor extractor) {
|
|
||||||
/* ---- optional data goes here: ---- */
|
/* ---- optional data goes here: ---- */
|
||||||
// If one of these fails, the frontend needs to handle that they are not available.
|
// If one of these fails, the frontend needs to handle that they are not available.
|
||||||
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
|
// Exceptions are therefore not thrown into the frontend, but stored into the error List,
|
||||||
|
@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
streamInfo.addException(e);
|
streamInfo.addException(e);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
StreamInfoItemCollector c = new StreamInfoItemCollector(
|
StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
|
||||||
extractor.getUrlIdHandler(), extractor.getServiceId());
|
|
||||||
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
|
StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
|
||||||
c.commit(nextVideo);
|
c.commit(nextVideo);
|
||||||
if (c.getItemList().size() != 0) {
|
if (c.getItemList().size() != 0) {
|
||||||
|
@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
|
||||||
return streamInfo;
|
return streamInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String uploader_thumbnail_url = "";
|
public void addException(Exception e) {
|
||||||
public String channel_url = "";
|
errors.add(e);
|
||||||
public String description = "";
|
}
|
||||||
|
|
||||||
public List<VideoStream> video_streams = null;
|
public StreamType stream_type;
|
||||||
public List<AudioStream> audio_streams = null;
|
public String uploader;
|
||||||
public List<VideoStream> video_only_streams = null;
|
public String thumbnail_url;
|
||||||
|
public String upload_date;
|
||||||
|
public long view_count = -1;
|
||||||
|
|
||||||
|
public String uploader_thumbnail_url;
|
||||||
|
public String channel_url;
|
||||||
|
public String description;
|
||||||
|
|
||||||
|
public List<VideoStream> video_streams;
|
||||||
|
public List<AudioStream> audio_streams;
|
||||||
|
public List<VideoStream> video_only_streams;
|
||||||
// video streams provided by the dash mpd do not need to be provided as VideoStream.
|
// video streams provided by the dash mpd do not need to be provided as VideoStream.
|
||||||
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
|
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
|
||||||
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
|
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
|
||||||
// providing the dash mpd fille will be possible in the future.
|
// providing the dash mpd fille will be possible in the future.
|
||||||
public String dashMpdUrl = "";
|
public String dashMpdUrl;
|
||||||
public int duration = -1;
|
public int duration = -1;
|
||||||
|
|
||||||
public int age_limit = -1;
|
public int age_limit = -1;
|
||||||
public int like_count = -1;
|
public int like_count = -1;
|
||||||
public int dislike_count = -1;
|
public int dislike_count = -1;
|
||||||
public String average_rating = "";
|
public String average_rating;
|
||||||
public StreamInfoItem next_video = null;
|
public StreamInfoItem next_video;
|
||||||
public List<InfoItem> related_streams = null;
|
public List<InfoItem> related_streams = new Vector<>();
|
||||||
//in seconds. some metadata is not passed using a StreamInfo object!
|
//in seconds. some metadata is not passed using a StreamInfo object!
|
||||||
public int start_position = 0;
|
public int start_position = 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
|
||||||
/**
|
/**
|
||||||
* Info object for previews of unopened videos, eg search results, related videos
|
* Info object for previews of unopened videos, eg search results, related videos
|
||||||
*/
|
*/
|
||||||
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem {
|
public class StreamInfoItem extends InfoItem {
|
||||||
public int duration;
|
public StreamType stream_type;
|
||||||
|
|
||||||
public InfoType infoType() {
|
public String uploader;
|
||||||
return InfoType.STREAM;
|
public String thumbnail_url;
|
||||||
}
|
public String upload_date;
|
||||||
|
public long view_count = -1;
|
||||||
|
public int duration = -1;
|
||||||
|
|
||||||
public String getTitle() {
|
public StreamInfoItem() {
|
||||||
return name;
|
super(InfoType.STREAM);
|
||||||
}
|
|
||||||
|
|
||||||
public String getLink() {
|
|
||||||
return url;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,8 +1,6 @@
|
||||||
package org.schabi.newpipe.extractor.stream;
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.InfoItemCollector;
|
import org.schabi.newpipe.extractor.InfoItemCollector;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
|
||||||
import org.schabi.newpipe.extractor.UrlIdHandler;
|
|
||||||
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
|
@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
|
||||||
public class StreamInfoItemCollector extends InfoItemCollector {
|
public class StreamInfoItemCollector extends InfoItemCollector {
|
||||||
|
|
||||||
private UrlIdHandler urlIdHandler;
|
public StreamInfoItemCollector(int serviceId) {
|
||||||
|
|
||||||
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
|
|
||||||
super(serviceId);
|
super(serviceId);
|
||||||
urlIdHandler = handler;
|
|
||||||
}
|
|
||||||
|
|
||||||
private UrlIdHandler getUrlIdHandler() {
|
|
||||||
return urlIdHandler;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
|
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
|
||||||
|
@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
|
||||||
// important information
|
// important information
|
||||||
resultItem.service_id = getServiceId();
|
resultItem.service_id = getServiceId();
|
||||||
resultItem.url = extractor.getWebPageUrl();
|
resultItem.url = extractor.getWebPageUrl();
|
||||||
if (getUrlIdHandler() == null) {
|
|
||||||
throw new ParsingException("Error: UrlIdHandler not set");
|
|
||||||
} else if (!resultItem.url.isEmpty()) {
|
|
||||||
resultItem.id = NewPipe.getService(getServiceId())
|
|
||||||
.getStreamUrlIdHandlerInstance()
|
|
||||||
.getId(resultItem.url);
|
|
||||||
}
|
|
||||||
resultItem.name = extractor.getTitle();
|
resultItem.name = extractor.getTitle();
|
||||||
resultItem.stream_type = extractor.getStreamType();
|
resultItem.stream_type = extractor.getStreamType();
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public interface StreamInfoItemExtractor {
|
public interface StreamInfoItemExtractor {
|
||||||
AbstractStreamInfo.StreamType getStreamType() throws ParsingException;
|
StreamType getStreamType() throws ParsingException;
|
||||||
String getWebPageUrl() throws ParsingException;
|
String getWebPageUrl() throws ParsingException;
|
||||||
String getTitle() throws ParsingException;
|
String getTitle() throws ParsingException;
|
||||||
int getDuration() throws ParsingException;
|
int getDuration() throws ParsingException;
|
||||||
|
|
10
stream/StreamType.java
Normal file
10
stream/StreamType.java
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
|
public enum StreamType {
|
||||||
|
NONE, // placeholder to check if stream type was checked or not
|
||||||
|
VIDEO_STREAM,
|
||||||
|
AUDIO_STREAM,
|
||||||
|
LIVE_STREAM,
|
||||||
|
AUDIO_LIVE_STREAM,
|
||||||
|
FILE
|
||||||
|
}
|
|
@ -1,7 +1,5 @@
|
||||||
package org.schabi.newpipe.extractor.stream;
|
package org.schabi.newpipe.extractor.stream;
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Created by Christian Schabesberger on 04.03.16.
|
* Created by Christian Schabesberger on 04.03.16.
|
||||||
*
|
*
|
||||||
|
@ -22,31 +20,24 @@ import java.io.Serializable;
|
||||||
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class VideoStream implements Serializable {
|
public class VideoStream extends Stream {
|
||||||
//url of the stream
|
public String resolution;
|
||||||
public String url = "";
|
public boolean isVideoOnly;
|
||||||
public int format = -1;
|
|
||||||
public String resolution = "";
|
|
||||||
public boolean isVideoOnly = false;
|
|
||||||
|
|
||||||
public VideoStream(String url, int format, String res) {
|
public VideoStream(String url, int format, String res) {
|
||||||
this(false, url, format, res);
|
this(url, format, res, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public VideoStream(boolean isVideoOnly, String url, int format, String res) {
|
public VideoStream(String url, int format, String res, boolean isVideoOnly) {
|
||||||
this.url = url;
|
super(url, format);
|
||||||
this.format = format;
|
|
||||||
this.resolution = res;
|
this.resolution = res;
|
||||||
this.isVideoOnly = isVideoOnly;
|
this.isVideoOnly = isVideoOnly;
|
||||||
}
|
}
|
||||||
|
|
||||||
// reveals whether two streams are the same, but have different urls
|
@Override
|
||||||
public boolean equalStats(VideoStream cmp) {
|
public boolean equalStats(Stream cmp) {
|
||||||
return format == cmp.format && resolution.equals(cmp.resolution);
|
return super.equalStats(cmp) && cmp instanceof VideoStream &&
|
||||||
}
|
resolution.equals(((VideoStream) cmp).resolution) &&
|
||||||
|
isVideoOnly == ((VideoStream) cmp).isVideoOnly;
|
||||||
// reveals whether two streams are equal
|
|
||||||
public boolean equals(VideoStream cmp) {
|
|
||||||
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.MediaFormat;
|
||||||
import org.schabi.newpipe.extractor.NewPipe;
|
import org.schabi.newpipe.extractor.NewPipe;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||||
|
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
|
||||||
import org.schabi.newpipe.extractor.stream.AudioStream;
|
import org.schabi.newpipe.extractor.stream.AudioStream;
|
||||||
|
import org.schabi.newpipe.extractor.stream.Stream;
|
||||||
|
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
||||||
|
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||||
import org.w3c.dom.Document;
|
import org.w3c.dom.Document;
|
||||||
import org.w3c.dom.Element;
|
import org.w3c.dom.Element;
|
||||||
import org.w3c.dom.NodeList;
|
import org.w3c.dom.NodeList;
|
||||||
|
@ -13,8 +17,6 @@ import org.w3c.dom.NodeList;
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Vector;
|
|
||||||
|
|
||||||
import javax.xml.parsers.DocumentBuilder;
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
|
@ -44,24 +46,25 @@ public class DashMpdParser {
|
||||||
private DashMpdParser() {
|
private DashMpdParser() {
|
||||||
}
|
}
|
||||||
|
|
||||||
static class DashMpdParsingException extends ParsingException {
|
public static class DashMpdParsingException extends ParsingException {
|
||||||
DashMpdParsingException(String message, Exception e) {
|
DashMpdParsingException(String message, Exception e) {
|
||||||
super(message, e);
|
super(message, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
|
/**
|
||||||
throws DashMpdParsingException, ReCaptchaException {
|
* Download manifest and return nodelist with elements of tag "AdaptationSet"
|
||||||
|
*/
|
||||||
|
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
|
||||||
String dashDoc;
|
String dashDoc;
|
||||||
Downloader downloader = NewPipe.getDownloader();
|
Downloader downloader = NewPipe.getDownloader();
|
||||||
try {
|
try {
|
||||||
dashDoc = downloader.download(dashManifestUrl);
|
dashDoc = downloader.download(streamInfo.dashMpdUrl);
|
||||||
} catch (IOException ioe) {
|
} catch (IOException ioe) {
|
||||||
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
|
throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.dashMpdUrl, ioe);
|
||||||
} catch (ReCaptchaException e) {
|
} catch (ReCaptchaException e) {
|
||||||
throw new ReCaptchaException("reCaptcha Challenge needed");
|
throw new ReCaptchaException("reCaptcha Challenge needed");
|
||||||
}
|
}
|
||||||
Vector<AudioStream> audioStreams = new Vector<>();
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||||
|
@ -69,27 +72,43 @@ public class DashMpdParser {
|
||||||
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
|
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
|
||||||
|
|
||||||
Document doc = builder.parse(stream);
|
Document doc = builder.parse(stream);
|
||||||
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet");
|
NodeList representationList = doc.getElementsByTagName("Representation");
|
||||||
for (int i = 0; i < adaptationSetList.getLength(); i++) {
|
|
||||||
Element adaptationSet = (Element) adaptationSetList.item(i);
|
for (int i = 0; i < representationList.getLength(); i++) {
|
||||||
String memeType = adaptationSet.getAttribute("mimeType");
|
Element representation = ((Element) representationList.item(i));
|
||||||
if (memeType.contains("audio")) {
|
try {
|
||||||
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0);
|
String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
|
||||||
|
String id = representation.getAttribute("id");
|
||||||
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
|
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
|
||||||
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth"));
|
ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
|
||||||
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate"));
|
if (itag != null) {
|
||||||
int format = -1;
|
MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
|
||||||
if (memeType.equals(MediaFormat.WEBMA.mimeType)) {
|
int format = mediaFormat != null ? mediaFormat.id : -1;
|
||||||
format = MediaFormat.WEBMA.id;
|
|
||||||
} else if (memeType.equals(MediaFormat.M4A.mimeType)) {
|
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
|
||||||
format = MediaFormat.M4A.id;
|
AudioStream audioStream = new AudioStream(url, format, itag.avgBitrate);
|
||||||
|
|
||||||
|
if (!Stream.containSimilarStream(audioStream, streamInfo.audio_streams)) {
|
||||||
|
streamInfo.audio_streams.add(audioStream);
|
||||||
}
|
}
|
||||||
audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate));
|
} else {
|
||||||
|
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
|
||||||
|
VideoStream videoStream = new VideoStream(url, format, itag.resolutionString, isVideoOnly);
|
||||||
|
|
||||||
|
if (isVideoOnly) {
|
||||||
|
if (!Stream.containSimilarStream(videoStream, streamInfo.video_only_streams)) {
|
||||||
|
streamInfo.video_only_streams.add(videoStream);
|
||||||
|
}
|
||||||
|
} else if (!Stream.containSimilarStream(videoStream, streamInfo.video_streams)) {
|
||||||
|
streamInfo.video_streams.add(videoStream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ignored) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new DashMpdParsingException("Could not parse Dash mpd", e);
|
throw new DashMpdParsingException("Could not parse Dash mpd", e);
|
||||||
}
|
}
|
||||||
return audioStreams;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
20
utils/Utils.java
Normal file
20
utils/Utils.java
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
package org.schabi.newpipe.extractor.utils;
|
||||||
|
|
||||||
|
public class Utils {
|
||||||
|
private Utils() {
|
||||||
|
//no instance
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove all non-digit characters from a string.<p>
|
||||||
|
* Examples:<br/>
|
||||||
|
* <ul><li>1 234 567 views -> 1234567</li>
|
||||||
|
* <li>$ 31,133.124 -> 31133124</li></ul>
|
||||||
|
*
|
||||||
|
* @param toRemove string to remove non-digit chars
|
||||||
|
* @return a string that contains only digits
|
||||||
|
*/
|
||||||
|
public static String removeNonDigitCharacters(String toRemove) {
|
||||||
|
return toRemove.replaceAll("\\D+", "");
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue