CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge TNP/dev into fynngodau/dev

Browse source
This commit is contained in:
Fynn Godau 2020-11-19 21:32:08 +01:00
commit 6bc7e3420e
80 changed files with 2542 additions and 2449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -11,7 +11,9 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.19.7'`the `dependencies` in your `build.gradle`. Replace `v0.19.7` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.3'`the `dependencies` in your `build.gradle`. Replace `v0.20.3` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
### Testing changes

6
build.gradle
View file

@ -2,10 +2,10 @@ allprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
sourceCompatibility = 1.7
targetCompatibility = 1.7
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.19.7'
version 'v0.20.3'
group 'com.github.TeamNewPipe'
repositories {

18
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View file

@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.util.Collections;
@ -277,18 +278,19 @@ public abstract class StreamingService {
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
* @param url the url on which it should be decided of which link type it is
* @return the link type of url
* @throws ParsingException
*/
public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
LinkHandlerFactory sH = getStreamLHFactory();
LinkHandlerFactory cH = getChannelLHFactory();
LinkHandlerFactory pH = getPlaylistLHFactory();
public final LinkType getLinkTypeByUrl(final String url) throws ParsingException {
final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
if (sH != null && sH.acceptUrl(url)) {
final LinkHandlerFactory sH = getStreamLHFactory();
final LinkHandlerFactory cH = getChannelLHFactory();
final LinkHandlerFactory pH = getPlaylistLHFactory();
if (sH != null && sH.acceptUrl(polishedUrl)) {
return LinkType.STREAM;
} else if (cH != null && cH.acceptUrl(url)) {
} else if (cH != null && cH.acceptUrl(polishedUrl)) {
return LinkType.CHANNEL;
} else if (pH != null && pH.acceptUrl(url)) {
} else if (pH != null && pH.acceptUrl(polishedUrl)) {
return LinkType.PLAYLIST;
} else {
return LinkType.NONE;

25
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java
View file

@ -42,12 +42,29 @@ public abstract class LinkHandlerFactory {
// Logic
///////////////////////////////////
public LinkHandler fromUrl(String url) throws ParsingException {
if (url == null) throw new IllegalArgumentException("url can not be null");
final String baseUrl = Utils.getBaseUrl(url);
return fromUrl(url, baseUrl);
/**
* Builds a {@link LinkHandler} from a url.<br>
* Be sure to call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
* this function.
* @param url the url to extract path and id from
* @return a {@link LinkHandler} complete with information
*/
public LinkHandler fromUrl(final String url) throws ParsingException {
final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
final String baseUrl = Utils.getBaseUrl(polishedUrl);
return fromUrl(polishedUrl, baseUrl);
}
/**
* Builds a {@link LinkHandler} from a url and a base url. The url is expected to be already
* polished from google search redirects (otherwise how could {@code baseUrl} have been
* extracted?).<br>
* So do not call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
* this function, since that should be done in {@link #fromUrl(String)}.
* @param url the url without google search redirects to extract id from
* @param baseUrl the base url
* @return a {@link LinkHandler} complete with information
*/
public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
if (url == null) throw new IllegalArgumentException("url can not be null");
if (!acceptUrl(url)) {

7
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java
View file

@ -31,9 +31,10 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
///////////////////////////////////
@Override
public ListLinkHandler fromUrl(String url) throws ParsingException {
String baseUrl = Utils.getBaseUrl(url);
return fromUrl(url, baseUrl);
public ListLinkHandler fromUrl(final String url) throws ParsingException {
final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
final String baseUrl = Utils.getBaseUrl(polishedUrl);
return fromUrl(polishedUrl, baseUrl);
}
@Override

46
extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java
View file

@ -3,30 +3,60 @@ package org.schabi.newpipe.extractor.localization;
import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Calendar;
import java.util.GregorianCalendar;
/**
* A wrapper class that provides a field to describe if the date is precise or just an approximation.
* A wrapper class that provides a field to describe if the date/time is precise or just an approximation.
*/
public class DateWrapper implements Serializable {
@NonNull private final Calendar date;
@NonNull private final OffsetDateTime offsetDateTime;
private final boolean isApproximation;
public DateWrapper(@NonNull Calendar date) {
this(date, false);
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar) {
this(calendar, false);
}
public DateWrapper(@NonNull Calendar date, boolean isApproximation) {
this.date = date;
/**
* @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead.
*/
@Deprecated
public DateWrapper(@NonNull Calendar calendar, boolean isApproximation) {
this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime) {
this(offsetDateTime, false);
}
public DateWrapper(@NonNull OffsetDateTime offsetDateTime, boolean isApproximation) {
this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC);
this.isApproximation = isApproximation;
}
/**
* @return the wrapped date.
* @return the wrapped date/time as a {@link Calendar}.
*
* @deprecated use {@link #offsetDateTime()} instead.
*/
@Deprecated
@NonNull
public Calendar date() {
return date;
return GregorianCalendar.from(offsetDateTime.toZonedDateTime());
}
/**
* @return the wrapped date/time.
*/
@NonNull
public OffsetDateTime offsetDateTime() {
return offsetDateTime;
}
/**

75
extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java
View file

@ -2,10 +2,11 @@ package org.schabi.newpipe.extractor.localization;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import org.schabi.newpipe.extractor.utils.Parser;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.Map;
import java.util.regex.Pattern;
@ -16,7 +17,7 @@ import java.util.regex.Pattern;
*/
public class TimeAgoParser {
private final PatternsHolder patternsHolder;
private final Calendar consistentNow;
private final OffsetDateTime now;
/**
* Creates a helper to parse upload dates in the format '2 days ago'.
@ -28,7 +29,7 @@ public class TimeAgoParser {
*/
public TimeAgoParser(PatternsHolder patternsHolder) {
this.patternsHolder = patternsHolder;
consistentNow = Calendar.getInstance();
now = OffsetDateTime.now(ZoneOffset.UTC);
}
/**
@ -42,14 +43,14 @@ public class TimeAgoParser {
* @throws ParsingException if the time unit could not be recognized
*/
public DateWrapper parse(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final TimeAgoUnit timeAgoUnit = caseUnitEntry.getKey();
for (Map.Entry<ChronoUnit, Map<String, Integer>> caseUnitEntry : patternsHolder.specialCases().entrySet()) {
final ChronoUnit chronoUnit = caseUnitEntry.getKey();
for (Map.Entry<String, Integer> caseMapToAmountEntry : caseUnitEntry.getValue().entrySet()) {
final String caseText = caseMapToAmountEntry.getKey();
final Integer caseAmount = caseMapToAmountEntry.getValue();
if (textualDateMatches(textualDate, caseText)) {
return getResultFor(caseAmount, timeAgoUnit);
return getResultFor(caseAmount, chronoUnit);
}
}
}
@ -63,8 +64,8 @@ public class TimeAgoParser {
timeAgoAmount = 1;
}
final TimeAgoUnit timeAgoUnit = parseTimeAgoUnit(textualDate);
return getResultFor(timeAgoAmount, timeAgoUnit);
final ChronoUnit chronoUnit = parseChronoUnit(textualDate);
return getResultFor(timeAgoAmount, chronoUnit);
}
private int parseTimeAgoAmount(String textualDate) throws NumberFormatException {
@ -72,13 +73,13 @@ public class TimeAgoParser {
return Integer.parseInt(timeValueStr);
}
private TimeAgoUnit parseTimeAgoUnit(String textualDate) throws ParsingException {
for (Map.Entry<TimeAgoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final TimeAgoUnit timeAgoUnit = entry.getKey();
private ChronoUnit parseChronoUnit(String textualDate) throws ParsingException {
for (Map.Entry<ChronoUnit, Collection<String>> entry : patternsHolder.asMap().entrySet()) {
final ChronoUnit chronoUnit = entry.getKey();
for (String agoPhrase : entry.getValue()) {
if (textualDateMatches(textualDate, agoPhrase)) {
return timeAgoUnit;
return chronoUnit;
}
}
}
@ -112,65 +113,35 @@ public class TimeAgoParser {
}
}
private DateWrapper getResultFor(int timeAgoAmount, TimeAgoUnit timeAgoUnit) {
final Calendar calendarTime = getNow();
private DateWrapper getResultFor(int timeAgoAmount, ChronoUnit chronoUnit) {
OffsetDateTime offsetDateTime = now;
boolean isApproximation = false;
switch (timeAgoUnit) {
switch (chronoUnit) {
case SECONDS:
calendarTime.add(Calendar.SECOND, -timeAgoAmount);
break;
case MINUTES:
calendarTime.add(Calendar.MINUTE, -timeAgoAmount);
break;
case HOURS:
calendarTime.add(Calendar.HOUR_OF_DAY, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
break;
case DAYS:
calendarTime.add(Calendar.DAY_OF_MONTH, -timeAgoAmount);
isApproximation = true;
break;
case WEEKS:
calendarTime.add(Calendar.WEEK_OF_YEAR, -timeAgoAmount);
isApproximation = true;
break;
case MONTHS:
calendarTime.add(Calendar.MONTH, -timeAgoAmount);
offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit);
isApproximation = true;
break;
case YEARS:
calendarTime.add(Calendar.YEAR, -timeAgoAmount);
// Prevent `PrettyTime` from showing '12 months ago'.
calendarTime.add(Calendar.DAY_OF_MONTH, -1);
// minusDays is needed to prevent `PrettyTime` from showing '12 months ago'.
offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1);
isApproximation = true;
break;
}
if (isApproximation) {
markApproximatedTime(calendarTime);
offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS);
}
return new DateWrapper(calendarTime, isApproximation);
}
private Calendar getNow() {
return (Calendar) consistentNow.clone();
}
/**
* Marks the time as approximated by setting minutes, seconds and milliseconds to 0.
*
* @param calendarTime Time to be marked as approximated
*/
private void markApproximatedTime(Calendar calendarTime) {
calendarTime.set(Calendar.MINUTE, 0);
calendarTime.set(Calendar.SECOND, 0);
calendarTime.set(Calendar.MILLISECOND, 0);
return new DateWrapper(offsetDateTime, isApproximation);
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View file

@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@ -71,8 +72,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray events = conferenceData.getArray("events");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray events = conferenceData.getArray("events");
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
}
@ -87,10 +88,11 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String conferenceUrl = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId();
try {
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
throw new ExtractionException("Could not parse json returnd by url: " + conferenceUrl);
}
}
@ -99,10 +101,4 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
public String getName() throws ParsingException {
return conferenceData.getString("title");
}
@Nonnull
@Override
public String getOriginalUrl() {
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
}
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java
View file

@ -2,25 +2,17 @@ package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
public final class MediaCCCParsingHelper {
private MediaCCCParsingHelper() { }
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
}

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java
View file

@ -19,14 +19,18 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
@ -93,7 +97,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getUploaderUrl() {
return data.getString("conference_url");
return MediaCCCConferenceLinkHandlerFactory.CONFERENCE_PATH + getUploaderName();
}
@Nonnull
@ -111,25 +115,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public String getDashMpdUrl() throws ParsingException {
public String getDashMpdUrl() {
return "";
}
@ -194,7 +198,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Override
public List<VideoStream> getVideoOnlyStreams() {
return null;
return Collections.emptyList();
}
@Nonnull
@ -214,9 +218,10 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
return StreamType.VIDEO_STREAM;
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedStreams() {
return new StreamInfoItemsCollector(getServiceId());
return null;
}
@Override
@ -227,14 +232,13 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String videoUrl = MediaCCCStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId();
try {
data = JsonParser.object().from(
downloader.get(getLinkHandler().getUrl()).responseBody());
data = JsonParser.object().from(downloader.get(videoUrl).responseBody());
conferenceData = JsonParser.object()
.from(downloader.get(getUploaderUrl()).responseBody());
.from(downloader.get(data.getString("conference_url")).responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json returned by url: "
+ getLinkHandler().getUrl(), jpe);
throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
}
}
@ -250,21 +254,25 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
return data.getString("frontend_link");
}
@Nonnull
@Override
public String getHost() {
return "";
}
@Nonnull
@Override
public String getPrivacy() {
return "";
}
@Nonnull
@Override
public String getCategory() {
return "";
}
@Nonnull
@Override
public String getLicence() {
return "";
@ -278,7 +286,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
return new ArrayList<>();
return Arrays.asList(data.getArray("tags").toArray(new String[0]));
}
@Nonnull

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java
View file

@ -7,30 +7,26 @@ import org.schabi.newpipe.extractor.utils.Parser;
import java.util.List;
public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/";
public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
@Override
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
throws ParsingException {
return "https://media.ccc.de/public/conferences/" + id;
public String getUrl(final String id,
final List<String> contentFilter,
final String sortFilter) throws ParsingException {
return CONFERENCE_PATH + id;
}
@Override
public String getId(final String url) throws ParsingException {
if (url.startsWith("https://media.ccc.de/public/conferences/")
|| url.startsWith("https://api.media.ccc.de/public/conferences/")) {
return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
} else if (url.startsWith("https://media.ccc.de/c/")) {
return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
} else if (url.startsWith("https://media.ccc.de/b/")) {
return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
}
throw new ParsingException("Could not get id from url: " + url);
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;
return getId(url) != null;
} catch (ParsingException e) {
return false;
}

45
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java
View file

@ -2,54 +2,27 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.net.MalformedURLException;
import java.net.URL;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://media.ccc.de/v/";
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
@Override
public String getId(final String urlString) throws ParsingException {
if (urlString.startsWith("https://media.ccc.de/public/events/")
&& !urlString.contains("?q=")) {
return urlString.substring(35); //remove …/public/events part
}
if (urlString.startsWith("https://api.media.ccc.de/public/events/")
&& !urlString.contains("?q=")) {
return urlString.substring(39); //remove api…/public/events part
}
URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String path = url.getPath();
// remove leading "/" of URL-path if URL-path is given
if (!path.isEmpty()) {
path = path.substring(1);
}
if (path.startsWith("v/")) {
return path.substring(2);
}
throw new ParsingException("Could not get id from url: " + url);
public String getId(final String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String getUrl(final String id) throws ParsingException {
return "https://media.ccc.de/public/events/" + id;
return VIDEO_PATH + id;
}
@Override
public boolean onAcceptUrl(final String url) {
try {
getId(url);
return true;
return getId(url) != null;
} catch (ParsingException e) {
return false;
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View file

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -12,11 +11,10 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
public class PeertubeParsingHelper {
public static final String START_KEY = "start";
@ -34,19 +32,12 @@ public class PeertubeParsingHelper {
}
}
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC);
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static Page getNextPage(final String prevPageUrl, final long total) {

34
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View file

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -13,17 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeAccountExtractor extends ChannelExtractor {
@ -85,14 +83,16 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
return getPage(new Page(
baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@ -122,8 +122,16 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
}
@Override
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
if (getId().contains("accounts/")) {
accountUrl += getId();
} else {
accountUrl += "accounts/" + getId();
}
final Response response = downloader.get(accountUrl);
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
@ -140,13 +148,9 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
if (json == null) throw new ExtractionException("Unable to extract PeerTube account data");
}
@Nonnull
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View file

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -13,17 +12,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -92,10 +90,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
return baseUrl + value;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
return getPage(new Page(
baseUrl + "/api/v1/" + getId() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
@ -130,7 +129,8 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
final Response response = downloader.get(
baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT + getId());
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
@ -147,13 +147,9 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
@Nonnull
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

73
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View file

@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
@ -37,11 +38,12 @@ import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
private final List<SubtitlesStream> subtitles = new ArrayList<>();
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
@ -64,11 +66,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + JsonUtils.getString(json, "previewPath");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
String text;
@ -81,7 +85,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
//if description is shortened, get full description
final Downloader dl = NewPipe.getDownloader();
try {
final Response response = dl.get(getUrl() + "/description");
final Response response = dl.get(baseUrl
+ PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+ getId() + "/description");
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
@ -107,9 +113,16 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public long getTimeStamp() {
//TODO fetch timestamp from url if present;
return 0;
public long getTimeStamp() throws ParsingException {
final long timestamp =
getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
if (timestamp == -2) {
// regex for timestamp was not found
return 0;
} else {
return timestamp;
}
}
@Override
@ -127,6 +140,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return json.getLong("dislikes");
}
@Nonnull
@Override
public String getUploaderUrl() throws ParsingException {
final String name = JsonUtils.getString(json, "account.name");
@ -134,11 +148,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Nonnull
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(json, "account.displayName");
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
String value;
@ -150,6 +166,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + value;
}
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
return JsonUtils.getString(json, "channel.url");
@ -173,11 +190,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return baseUrl + value;
}
@Nonnull
@Override
public String getDashMpdUrl() {
return "";
}
@Nonnull
@Override
public String getHlsUrl() {
return "";
@ -185,7 +204,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public List<AudioStream> getAudioStreams() {
return null;
return Collections.emptyList();
}
@Override
@ -220,11 +239,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return Collections.emptyList();
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitlesDefault() {
return subtitles;
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
final List<SubtitlesStream> filteredSubs = new ArrayList<>();
@ -241,21 +262,27 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return StreamType.VIDEO_STREAM;
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final List<String> tags = getTags();
final String apiUrl;
if (!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags);
} else {
if (tags.isEmpty()) {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
} else {
apiUrl = getRelatedStreamsUrl(tags);
}
if (Utils.isBlank(apiUrl)) {
return null;
} else {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
getStreamsFromApi(collector, apiUrl);
return collector;
}
if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
return collector;
}
@Nonnull
@Override
public List<String> getTags() {
try {
@ -327,7 +354,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
final Response response = downloader.get(baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId());
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
@ -343,14 +370,18 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract PeerTube stream data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube stream data");
if (json == null) {
throw new ExtractionException("Unable to extract PeerTube stream data");
}
PeertubeParsingHelper.validate(json);
}
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
final Response response = getDownloader().get(getUrl() + "/captions");
final Response response = getDownloader().get(baseUrl
+ PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+ getId() + "/captions");
final JsonObject captionsJson = JsonParser.object().from(response.responseBody());
final JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for (final Object c : captions) {
@ -370,31 +401,31 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "name");
}
@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/videos/watch/" + getId();
}
@Nonnull
@Override
public String getHost() throws ParsingException {
return JsonUtils.getString(json, "account.host");
}
@Nonnull
@Override
public String getPrivacy() throws ParsingException {
return JsonUtils.getString(json, "privacy.label");
}
@Nonnull
@Override
public String getCategory() throws ParsingException {
return JsonUtils.getString(json, "category.label");
}
@Nonnull
@Override
public String getLicence() throws ParsingException {
return JsonUtils.getString(json, "licence.label");

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View file

@ -27,8 +27,7 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
@Override
public String getThumbnailUrl() throws ParsingException {
final String value = JsonUtils.getString(item, "thumbnailPath");
return baseUrl + value;
return baseUrl + JsonUtils.getString(item, "thumbnailPath");
}
@Override
@ -51,7 +50,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
final String name = JsonUtils.getString(item, "account.name");
final String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory()
.fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java
View file

@ -11,7 +11,7 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
private static final String API_ENDPOINT = "/api/v1/";
public static final String API_ENDPOINT = "/api/v1/";
public static PeertubeChannelLinkHandlerFactory getInstance() {
return instance;
@ -24,19 +24,17 @@ public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
@Override
public String getUrl(String id, List<String> contentFilters, String searchFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, contentFilters, searchFilter, baseUrl);
return getUrl(id, contentFilters, searchFilter, ServiceList.PeerTube.getBaseUrl());
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
if (id.matches(ID_PATTERN)) {
return baseUrl + API_ENDPOINT + id;
return baseUrl + "/" + id;
} else {
// This is needed for compatibility with older versions were we didn't support video channels yet
return baseUrl + API_ENDPOINT + "accounts/" + id;
return baseUrl + "/accounts/" + id;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeStreamLinkHandlerFactory.java
View file

@ -10,7 +10,8 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final PeertubeStreamLinkHandlerFactory instance = new PeertubeStreamLinkHandlerFactory();
private static final String ID_PATTERN = "/videos/(watch/|embed/)?([^/?&#]*)";
private static final String VIDEO_ENDPOINT = "/api/v1/videos/";
public static final String VIDEO_API_ENDPOINT = "/api/v1/videos/";
private static final String VIDEO_PATH = "/videos/watch/";
private PeertubeStreamLinkHandlerFactory() {
}
@ -21,13 +22,12 @@ public class PeertubeStreamLinkHandlerFactory extends LinkHandlerFactory {
@Override
public String getUrl(String id) {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
return getUrl(id, baseUrl);
return getUrl(id, ServiceList.PeerTube.getBaseUrl());
}
@Override
public String getUrl(String id, String baseUrl) {
return baseUrl + VIDEO_ENDPOINT + id;
return baseUrl + VIDEO_PATH + id;
}
@Override

43
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file

@ -25,10 +25,15 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
@ -93,23 +98,16 @@ public class SoundcloudParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e1) {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e1) {
try {
date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(textualUploadDate);
} catch (ParseException e2) {
return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"));
} catch (DateTimeParseException e2) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" + ", " + e1.getMessage(), e2);
}
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
/**
@ -148,12 +146,21 @@ public class SoundcloudParsingHelper {
*
* @return the resolved id
*/
public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e){
throw new IllegalArgumentException("The given URL is not valid");
}
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
}

100
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View file

@ -33,6 +33,7 @@ import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -46,7 +47,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
String policy = track.getString("policy", EMPTY_STRING);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
@ -68,8 +69,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getTextualUploadDate() throws ParsingException {
return track.getString("created_at").replace("T"," ").replace("Z", "");
public String getTextualUploadDate() {
return track.getString("created_at")
.replace("T"," ")
.replace("Z", "");
}
@Nonnull
@ -85,10 +88,10 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
if (artworkUrl.isEmpty()) {
artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
return artworkUrl.replace("large.jpg", "crop.jpg");
}
@Nonnull
@Override
public Description getDescription() {
return new Description(track.getString("description"), Description.PLAIN_TEXT);
@ -144,19 +147,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@ -168,14 +171,14 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getHlsUrl() throws ParsingException {
public String getHlsUrl() {
return "";
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
List<AudioStream> audioStreams = new ArrayList<>();
Downloader dl = NewPipe.getDownloader();
final Downloader dl = NewPipe.getDownloader();
// Streams can be streamable and downloadable - or explicitly not.
// For playing the track, it is only necessary to have a streamable track.
@ -183,12 +186,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
if (!track.getBoolean("streamable")) return audioStreams;
try {
JsonArray transcodings = track.getObject("media").getArray("transcodings");
final JsonArray transcodings = track.getObject("media").getArray("transcodings");
// get information about what stream formats are available
for (Object transcoding : transcodings) {
JsonObject t = (JsonObject) transcoding;
final JsonObject t = (JsonObject) transcoding;
String url = t.getString("url");
if (!isNullOrEmpty(url)) {
@ -200,7 +203,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
// This url points to the endpoint which generates a unique and short living url to the stream.
// TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream)
url += "?client_id=" + SoundcloudParsingHelper.clientId();
String res = dl.get(url).responseBody();
final String res = dl.get(url).responseBody();
try {
JsonObject mp3UrlObject = JsonParser.object().from(res);
@ -234,24 +237,24 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
return null;
public List<VideoStream> getVideoStreams() {
return Collections.emptyList();
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
return null;
}
@Override
@Nonnull
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
public List<VideoStream> getVideoOnlyStreams() {
return Collections.emptyList();
}
@Override
@Nonnull
public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException {
public List<SubtitlesStream> getSubtitlesDefault() {
return Collections.emptyList();
}
@Override
@Nonnull
public List<SubtitlesStream> getSubtitles(MediaFormat format) {
return Collections.emptyList();
}
@ -260,12 +263,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return StreamType.AUDIO_STREAM;
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
final String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
+ "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
return collector;
@ -276,40 +280,44 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return null;
}
@Nonnull
@Override
public String getHost() throws ParsingException {
public String getHost() {
return "";
}
@Nonnull
@Override
public String getPrivacy() {
return "";
}
@Nonnull
@Override
public String getCategory() {
return "";
}
@Nonnull
@Override
public String getLicence() {
return "";
}
@Override
public String getPrivacy() throws ParsingException {
return "";
}
@Override
public String getCategory() throws ParsingException {
return "";
}
@Override
public String getLicence() throws ParsingException {
return "";
}
@Override
public Locale getLanguageInfo() throws ParsingException {
public Locale getLanguageInfo() {
return null;
}
@Nonnull
@Override
public List<String> getTags() throws ParsingException {
return new ArrayList<>();
public List<String> getTags() {
return Collections.emptyList();
}
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
public String getSupportInfo() {
return "";
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java
View file

@ -94,7 +94,7 @@ public class ItagItem {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
throw new ParsingException("itag=" + itagId + " not supported");
}
/*//////////////////////////////////////////////////////////////////////////

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file

@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response;
@ -22,13 +21,20 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -55,12 +61,6 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}
/**
* The official youtube app supports intents in this format, where after the ':' is the videoId.
* Accordingly there are other apps sharing streams in this format.
*/
public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
@ -182,23 +182,27 @@ public class YoutubeParsingHelper {
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
try {
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
} catch (ParseException e) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
try {
return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC);
} catch (DateTimeParseException e1) {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1);
}
}
final Calendar uploadDate = Calendar.getInstance();
uploadDate.setTime(date);
return uploadDate;
}
public static JsonObject getInitialData(String html) throws ParsingException {
try {
String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
} catch (Parser.RegexException e) {
final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html);
return JsonParser.object().from(initialData);
}
} catch (JsonParserException | Parser.RegexException e) {
throw new ParsingException("Could not get ytInitialData", e);
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -18,9 +17,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
@ -87,7 +85,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL") && !browseId.isEmpty()) {
if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
@ -191,12 +190,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
// If there's no subscribe button, the channel has the subscriber count disabled
if (c4TabbedHeaderRenderer.has("subscribeButton")) {
return 0;
} else {
return -1;
}
return ITEM_COUNT_UNKNOWN;
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View file

@ -47,13 +47,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
return getPage(getNextPage(commentsToken));
}
private Page getNextPage(JsonObject ajaxJson) throws ParsingException {
JsonArray arr;
final JsonArray arr;
try {
arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
} catch (Exception e) {
@ -89,14 +89,14 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
String ajaxResponse = makeAjaxRequest(page.getUrl());
JsonObject ajaxJson;
final String ajaxResponse = makeAjaxRequest(page.getUrl());
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
} catch (Exception e) {
throw new ParsingException("Could not parse json data for comments", e);
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectCommentsFrom(collector, ajaxJson);
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
}
@ -160,8 +160,8 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
private String findValue(String doc, String start, String end) {
int beginIndex = doc.indexOf(start) + start.length();
int endIndex = doc.indexOf(end, beginIndex);
final int beginIndex = doc.indexOf(start) + start.length();
final int endIndex = doc.indexOf(end, beginIndex);
return doc.substring(beginIndex, endIndex);
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View file

@ -34,7 +34,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getThumbnailUrl() throws ParsingException {
try {
JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return JsonUtils.getString(arr.getObject(2), "url");
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
@ -82,7 +82,13 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getCommentText() throws ParsingException {
try {
String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText"));
final JsonObject contentText = JsonUtils.getObject(json, "contentText");
if (contentText.isEmpty()) {
// completely empty comments as described in
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
return "";
}
final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
return Utils.removeUTF8BOM(commentText);
} catch (Exception e) {

19
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java
View file

@ -7,11 +7,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
private final Element entryElement;
@ -62,19 +59,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
final Date date;
try {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+00:00");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
date = dateFormat.parse(getTextualUploadDate());
} catch (ParseException e) {
return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate()));
} catch (DateTimeParseException e) {
throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", e);
}
final Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
return new DateWrapper(calendar);
}
@Override

588
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -3,7 +3,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
@ -36,22 +41,21 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@ -84,27 +88,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Exceptions
//////////////////////////////////////////////////////////////////////////*/
public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) {
public static class DeobfuscateException extends ParsingException {
DeobfuscateException(String message, Throwable cause) {
super(message, cause);
}
}
/*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
private JsonArray initialAjaxJson;
@Nullable
private JsonObject playerArgs;
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit;
@Nonnull
private List<SubtitlesInfo> subtitlesInfos = new ArrayList<>();
private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
@ -135,18 +137,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return title;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
return null;
}
JsonObject micro = playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (micro.isString("uploadDate") && !micro.getString("uploadDate").isEmpty()) {
final JsonObject micro =
playerResponse.getObject("microformat").getObject("playerMicroformatRenderer");
if (!micro.getString("uploadDate", EMPTY_STRING).isEmpty()) {
return micro.getString("uploadDate");
}
if (micro.isString("publishDate") && !micro.getString("publishDate").isEmpty()) {
} else if (!micro.getString("publishDate", EMPTY_STRING).isEmpty()) {
return micro.getString("publishDate");
} else {
final JsonObject liveDetails = micro.getObject("liveBroadcastDetails");
if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
// an ended live stream
return liveDetails.getString("endTimestamp");
} else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
// a running live stream
return liveDetails.getString("startTimestamp");
} else if (getStreamType() == StreamType.LIVE_STREAM) {
// this should never be reached, but a live stream without upload date is valid
return null;
}
}
if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")).startsWith("Premiered")) {
@ -154,22 +165,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { // Premiered 20 hours ago
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime();
return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime);
} catch (Exception ignored) {
}
try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
final LocalDate localDate = LocalDate.parse(time,
DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
}
try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {}
LocalDate localDate = LocalDate.parse(getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")),
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
} catch (Exception ignored) {
}
throw new ParsingException("Could not get upload date");
}
@ -217,9 +233,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Override
public int getAgeLimit() {
if (isNullOrEmpty(initialData)) throw new IllegalStateException("initialData is not parsed yet");
public int getAgeLimit() throws ParsingException {
if (ageLimit == -1) {
ageLimit = NO_AGE_LIMIT;
final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
.getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
.getArray("rows");
for (final Object metadataRow : metadataRows) {
final JsonArray contents = ((JsonObject) metadataRow)
.getObject("metadataRowRenderer").getArray("contents");
for (final Object content : contents) {
final JsonArray runs = ((JsonObject) content).getArray("runs");
for (final Object run : runs) {
final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
if (rowText.contains("Age-restricted")) {
ageLimit = 18;
return ageLimit;
}
}
}
}
}
return ageLimit;
}
@ -253,7 +288,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
*/
@Override
public long getTimeStamp() throws ParsingException {
return getTimestampSeconds("((#|&|\\?)(t|start)=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
final long timestamp =
getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
if (timestamp == -2) {
// regex for timestamp was not found
return 0;
} else {
return timestamp;
}
}
@Override
@ -298,8 +341,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + likesString + "\" as an Integer", nfe);
} catch (Exception e) {
if (ageLimit == 18) return -1;
throw new ParsingException("Could not get like count", e);
if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get like count", e);
}
return -1;
}
}
@ -322,8 +367,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (NumberFormatException nfe) {
throw new ParsingException("Could not parse \"" + dislikesString + "\" as an Integer", nfe);
} catch (Exception e) {
if (ageLimit == 18) return -1;
throw new ParsingException("Could not get dislike count", e);
if (getAgeLimit() == NO_AGE_LIMIT) {
throw new ParsingException("Could not get dislike count", e);
}
return -1;
}
}
@ -360,7 +407,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (ParsingException ignored) { }
} catch (ParsingException ignored) {
}
if (isNullOrEmpty(uploaderName)) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
@ -386,8 +434,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
if (isNullOrEmpty(url)) {
if (ageLimit == 18) return "";
throw new ParsingException("Could not get uploader avatar URL");
if (ageLimit == NO_AGE_LIMIT) {
throw new ParsingException("Could not get uploader avatar URL");
}
return "";
}
return fixThumbnailUrl(url);
@ -395,19 +445,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@ -421,18 +471,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return playerResponse.getObject("streamingData").getString("dashManifestUrl");
} else if (videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs != null && playerArgs.isString("dashmpd")) {
dashManifestUrl = playerArgs.getString("dashmpd", EMPTY_STRING);
} else {
return "";
}
if (!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig;
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String deobfuscatedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
deobfuscatedSig = deobfuscateSignature(obfuscatedSig);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
}
return dashManifestUrl;
@ -449,11 +497,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
return playerResponse.getObject("streamingData").getString("hlsManifestUrl");
} catch (Exception e) {
if (playerArgs != null && playerArgs.isString("hlsvp")) {
return playerArgs.getString("hlsvp");
} else {
throw new ParsingException("Could not get hls manifest url", e);
}
throw new ParsingException("Could not get hls manifest url", e);
}
}
@ -519,35 +563,57 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
@Nonnull
public List<SubtitlesStream> getSubtitlesDefault() {
public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {
return getSubtitles(MediaFormat.TTML);
}
@Override
@Nonnull
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws ParsingException {
assertPageFetched();
List<SubtitlesStream> subtitles = new ArrayList<>();
for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
subtitles.add(subtitlesInfo.getSubtitle(format));
// If the video is age restricted getPlayerConfig will fail
if (getAgeLimit() != NO_AGE_LIMIT) {
return Collections.emptyList();
}
if (subtitles != null) {
// already calculated
return subtitles;
}
final JsonObject renderer = playerResponse.getObject("captions")
.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// TODO: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
subtitles = new ArrayList<>();
for (int i = 0; i < captionsArray.size(); i++) {
final String languageCode = captionsArray.getObject(i).getString("languageCode");
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
final String vssId = captionsArray.getObject(i).getString("vssId");
if (languageCode != null && baseUrl != null && vssId != null) {
final boolean isAutoGenerated = vssId.startsWith("a.");
final String cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
subtitles.add(new SubtitlesStream(format, languageCode,
cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated));
}
}
return subtitles;
}
@Override
public StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() {
assertPageFetched();
try {
if (!playerResponse.getObject("streamingData").has(FORMATS) ||
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
return StreamType.LIVE_STREAM;
}
} catch (Exception e) {
throw new ParsingException("Could not get stream type", e);
}
return StreamType.VIDEO_STREAM;
return playerResponse.getObject("streamingData").has(FORMATS)
? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM;
}
@Nullable
private StreamInfoItemExtractor getNextStream() throws ExtractionException {
try {
final JsonObject firstWatchNextItem = initialData.getObject("contents")
@ -568,11 +634,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
assertPageFetched();
if (getAgeLimit() != NO_AGE_LIMIT) return null;
if (getAgeLimit() != NO_AGE_LIMIT) {
return null;
}
try {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
@ -604,10 +673,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public String getErrorMessage() {
try {
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
.getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
} catch (ParsingException e) {
return null;
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse")
.getObject("playabilityStatus").getObject("errorScreen")
.getObject("playerErrorMessageRenderer").getObject("reason"));
} catch (ParsingException | NullPointerException e) {
return null; // no error message
}
}
@ -618,7 +688,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";
private final static String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
@ -627,36 +697,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(",
"\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("
};
;
private volatile String decryptionCode = "";
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
initialAjaxJson = getJsonResponse(getUrl() + "&pbj=1", getExtractorLocalization());
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());
final String playerUrl;
if (initialAjaxJson.getObject(2).has("response")) { // age-restricted videos
initialData = initialAjaxJson.getObject(2).getObject("response");
ageLimit = 18;
final EmbeddedInfo info = getEmbeddedInfo();
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
playerUrl = info.url;
} else {
initialData = initialAjaxJson.getObject(3).getObject("response");
ageLimit = NO_AGE_LIMIT;
playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
initialData = initialAjaxJson.getObject(3).getObject("response", null);
if (initialData == null) {
initialData = initialAjaxJson.getObject(2).getObject("response", null);
if (initialData == null) {
throw new ParsingException("Could not get initial data");
}
}
playerResponse = getPlayerResponse();
playerResponse = initialAjaxJson.getObject(2).getObject("playerResponse", null);
if (playerResponse == null || !playerResponse.has("streamingData")) {
// try to get player response by fetching video info page
fetchVideoInfoPage();
}
final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
final String status = playabilityStatus.getString("status");
@ -665,146 +724,59 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String reason = playabilityStatus.getString("reason");
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
if (subtitlesInfos.isEmpty()) {
subtitlesInfos.addAll(getAvailableSubtitlesInfo());
}
}
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
JsonObject playerArgs;
private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
final String infoPageResponse = NewPipe.getDownloader()
.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
//attempt to load the youtube js player JSON arguments
try {
playerArgs = playerConfig.getObject("args");
} catch (Exception e) {
throw new ParsingException("Could not parse yt player config", e);
}
return playerArgs;
}
private String getPlayerUrl(JsonObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl;
JsonObject ytAssets = playerConfig.getObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (Exception e) {
throw new ParsingException("Could not load decryption code for the Youtube service.", e);
}
}
private JsonObject getPlayerResponse() throws ParsingException {
try {
String playerResponseStr;
if (playerArgs != null) {
playerResponseStr = playerArgs.getString("player_response");
} else {
playerResponseStr = videoInfoPage.get("player_response");
}
return JsonParser.object().from(playerResponseStr);
} catch (Exception e) {
throw new ParsingException("Could not parse yt player response", e);
playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
} catch (JsonParserException e) {
throw new ParsingException(
"Could not parse YouTube player response from video info page", e);
}
}
@Nonnull
private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException {
private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
try {
final Downloader downloader = NewPipe.getDownloader();
final String embedUrl = "https://www.youtube.com/embed/" + getId();
final String embedPageContent = downloader.get(embedUrl, getExtractorLocalization()).responseBody();
// Get player url
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
String playerUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
final String embedPageContent = NewPipe.getDownloader()
.get(embedUrl, getExtractorLocalization()).responseBody();
try {
// Get embed sts
final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
return new EmbeddedInfo(playerUrl, sts);
} catch (Exception i) {
// if it fails we simply reply with no sts as then it does not seem to be necessary
return new EmbeddedInfo(playerUrl, "");
final String assetsPattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")";
playerJsUrl = Parser.matchGroup1(assetsPattern, embedPageContent)
.replace("\\", "").replace("\"", "");
} catch (Parser.RegexException ex) {
// playerJsUrl is still available in the file, just somewhere else TODO
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (Element elem : elems) {
if (elem.attr("src").contains("base.js")) {
playerJsUrl = elem.attr("src");
break;
}
}
}
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
// Get embed sts
return Parser.matchGroup1("\"sts\"\\s*:\\s*(\\d+)", embedPageContent);
} catch (Exception i) {
// if it fails we simply reply with no sts as then it does not seem to be necessary
return "";
}
}
private String loadDecryptionCode(String playerUrl) throws DecryptException {
try {
Downloader downloader = NewPipe.getDownloader();
if (!playerUrl.contains("https://youtube.com")) {
//sometimes the https://youtube.com part does not get send with
//than we have to add it by hand
playerUrl = "https://youtube.com" + playerUrl;
}
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
final String functionPattern = "("
+ decryptionFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction);
final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
final String callerFunction =
"function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}";
return helperObject + decryptionFunction + callerFunction;
} catch (IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch (Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
}
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result;
try {
ScriptableObject scope = context.initStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) {
throw new DecryptException("could not get decrypt signature", e);
} finally {
Context.exit();
}
return result == null ? "" : result.toString();
}
private String getDecryptionFuncName(final String playerCode) throws DecryptException {
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
@ -815,75 +787,81 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
}
throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception);
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
}
private String loadDeobfuscationCode(@Nonnull final String playerJsUrl)
throws DeobfuscateException {
try {
final String playerCode = NewPipe.getDownloader()
.get(playerJsUrl, getExtractorLocalization()).responseBody();
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
final String functionPattern = "("
+ deobfuscationFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction);
final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
final String callerFunction =
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";
return helperObject + deobfuscateFunction + callerFunction;
} catch (IOException ioe) {
throw new DeobfuscateException("Could not load deobfuscate function", ioe);
} catch (Exception e) {
throw new DeobfuscateException("Could not parse deobfuscate function ", e);
}
}
@Nonnull
private List<SubtitlesInfo> getAvailableSubtitlesInfo() {
// If the video is age restricted getPlayerConfig will fail
if (getAgeLimit() != NO_AGE_LIMIT) return Collections.emptyList();
final JsonObject captions;
if (!playerResponse.has("captions")) {
// Captions does not exist
return Collections.emptyList();
}
captions = playerResponse.getObject("captions");
final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer");
final JsonArray captionsArray = renderer.getArray("captionTracks");
// todo: use this to apply auto translation to different language from a source language
// final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
// This check is necessary since there may be cases where subtitles metadata do not contain caption track info
// e.g. https://www.youtube.com/watch?v=-Vpwatutnko
final int captionsSize = captionsArray.size();
if (captionsSize == 0) return Collections.emptyList();
List<SubtitlesInfo> result = new ArrayList<>();
for (int i = 0; i < captionsSize; i++) {
final String languageCode = captionsArray.getObject(i).getString("languageCode");
final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
final String vssId = captionsArray.getObject(i).getString("vssId");
if (languageCode != null && baseUrl != null && vssId != null) {
final boolean isAutoGenerated = vssId.startsWith("a.");
result.add(new SubtitlesInfo(baseUrl, languageCode, isAutoGenerated));
private String getDeobfuscationCode() throws ParsingException {
if (cachedDeobfuscationCode == null) {
if (playerJsUrl == null) {
// the currentPlayerJsUrl was not found in any page fetched so far and there is
// nothing cached, so try fetching embedded info
getEmbeddedInfoStsAndStorePlayerJsUrl();
if (playerJsUrl == null) {
throw new ParsingException(
"Embedded info did not provide YouTube player js url");
}
}
if (playerJsUrl.startsWith("//")) {
playerJsUrl = HTTPS + playerJsUrl;
} else if (playerJsUrl.startsWith("/")) {
// sometimes https://youtube.com part has to be added manually
playerJsUrl = HTTPS + "//youtube.com" + playerJsUrl;
}
cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl);
}
return result;
}
/*//////////////////////////////////////////////////////////////////////////
// Data Class
//////////////////////////////////////////////////////////////////////////*/
private class EmbeddedInfo {
final String url;
final String sts;
EmbeddedInfo(final String url, final String sts) {
this.url = url;
this.sts = sts;
}
return cachedDeobfuscationCode;
}
private class SubtitlesInfo {
final String cleanUrl;
final String languageCode;
final boolean isGenerated;
private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException {
final String deobfuscationCode = getDeobfuscationCode();
public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
this.cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language
this.languageCode = languageCode;
this.isGenerated = isGenerated;
}
public SubtitlesStream getSubtitle(final MediaFormat format) {
return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated);
final Context context = Context.enter();
context.setOptimizationLevel(-1);
final Object result;
try {
final ScriptableObject scope = context.initSafeStandardObjects();
context.evaluateString(scope, deobfuscationCode, "deobfuscationCode", 1, null);
final Function deobfuscateFunc = (Function) scope.get(DEOBFUSCATION_FUNC_NAME, scope);
result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig});
} catch (Exception e) {
throw new DeobfuscateException("Could not get deobfuscate signature", e);
} finally {
Context.exit();
}
return result == null ? "" : result.toString();
}
/*//////////////////////////////////////////////////////////////////////////
@ -942,14 +920,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"&sts=" + sts + "&ps=default&gl=US&hl=en";
}
private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
JsonObject streamingData = playerResponse.getObject("streamingData");
private Map<String, ItagItem> getItags(final String streamingDataKey,
final ItagItem.ItagType itagTypeWanted)
throws ParsingException {
final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
final JsonObject streamingData = playerResponse.getObject("streamingData");
if (!streamingData.has(streamingDataKey)) {
return urlAndItags;
}
JsonArray formats = streamingData.getArray(streamingDataKey);
final JsonArray formats = streamingData.getArray(streamingDataKey);
for (int i = 0; i != formats.size(); ++i) {
JsonObject formatData = formats.getObject(i);
int itag = formatData.getInt("itag");
@ -958,22 +938,30 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try {
ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == itagTypeWanted) {
// Ignore streams that are delivered using YouTube's OTF format,
// as those only work with DASH and not with progressive HTTP.
if (formatData.getString("type", EMPTY_STRING)
.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
continue;
}
String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// this url has an encrypted signature
// this url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ decryptSignature(cipher.get("s"), decryptionCode);
+ deobfuscateSignature(cipher.get("s"));
}
urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {}
} catch (UnsupportedEncodingException ignored) {
}
}
}
@ -984,12 +972,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override
public List<Frameset> getFrames() throws ExtractionException {
try {
JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
final String resp = jo.getObject("args").getString("player_response");
jo = JsonParser.object().from(resp);
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
final JsonObject storyboards = playerResponse.getObject("storyboards");
final JsonObject storyboardsRenderer;
if (storyboards.has("playerLiveStoryboardSpecRenderer")) {
storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer");
} else {
storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer");
}
final String[] spec = storyboardsRenderer.getString("spec").split("\\|");
final String url = spec[0];
final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8) {
@ -1059,7 +1053,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<String> getTags() {
return new ArrayList<>();
return Collections.emptyList();
}
@Nonnull

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View file

@ -12,11 +12,14 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -165,8 +168,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
if (isPremiere()) {
final Date date = getDateFromPremiere().getTime();
return new SimpleDateFormat("yyyy-MM-dd HH:mm").format(date);
return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere());
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
@ -250,15 +252,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return videoInfo.has("upcomingEventData");
}
private Calendar getDateFromPremiere() throws ParsingException {
private OffsetDateTime getDateFromPremiere() throws ParsingException {
final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData");
final String startTime = upcomingEventData.getString("startTime");
try {
final long startTimeTimestamp = Long.parseLong(startTime);
final Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(startTimeTimestamp * 1000L));
return calendar;
return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)),
ZoneOffset.UTC);
} catch (Exception e) {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
}

129
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java
View file

@ -1,126 +1,71 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.INPUT_STREAM;
/**
* Extract subscriptions from a YouTube export (OPML format supported)
* Extract subscriptions from a Google takout export (the user has to get the JSON out of the zip)
*/
public class YoutubeSubscriptionExtractor extends SubscriptionExtractor {
private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
public YoutubeSubscriptionExtractor(YoutubeService service) {
super(service, Collections.singletonList(INPUT_STREAM));
public YoutubeSubscriptionExtractor(final YoutubeService youtubeService) {
super(youtubeService, Collections.singletonList(INPUT_STREAM));
}
@Override
public String getRelatedUrl() {
return "https://www.youtube.com/subscription_manager?action_takeout=1";
return "https://takeout.google.com/takeout/custom/youtube";
}
@Override
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws ExtractionException {
if (contentInputStream == null) throw new InvalidSourceException("input stream is null");
return getItemsFromOPML(contentInputStream);
}
/*//////////////////////////////////////////////////////////////////////////
// OPML implementation
//////////////////////////////////////////////////////////////////////////*/
private static final String ID_PATTERN = "/videos.xml\\?channel_id=([A-Za-z0-9_-]*)";
private static final String BASE_CHANNEL_URL = "https://www.youtube.com/channel/";
private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream) throws ExtractionException {
final List<SubscriptionItem> result = new ArrayList<>();
final String contentString = readFromInputStream(contentInputStream);
Document document = Jsoup.parse(contentString, "", org.jsoup.parser.Parser.xmlParser());
if (document.select("opml").isEmpty()) {
throw new InvalidSourceException("document does not have OPML tag");
}
if (document.select("outline").isEmpty()) {
throw new InvalidSourceException("document does not have at least one outline tag");
}
for (Element outline : document.select("outline[type=rss]")) {
String title = outline.attr("title");
String xmlUrl = outline.attr("abs:xmlUrl");
try {
String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
} catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
}
return result;
}
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
/**
* Throws an exception if the string does not have the right tag/string from a valid export.
*/
private void throwIfTagIsNotFound(String content) throws InvalidSourceException {
if (!content.trim().contains("<opml")) {
throw new InvalidSourceException("input stream does not have OPML tag");
}
}
private String readFromInputStream(InputStream inputStream) throws InvalidSourceException {
StringBuilder contentBuilder = new StringBuilder();
boolean hasTag = false;
public List<SubscriptionItem> fromInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException {
final JsonArray subscriptions;
try {
byte[] buffer = new byte[16 * 1024];
int read;
while ((read = inputStream.read(buffer)) != -1) {
String currentPartOfContent = new String(buffer, 0, read, "UTF-8");
contentBuilder.append(currentPartOfContent);
subscriptions = JsonParser.array().from(contentInputStream);
} catch (JsonParserException e) {
throw new InvalidSourceException("Invalid json input stream", e);
}
// Fail-fast in case of reading a long unsupported input stream
if (!hasTag && contentBuilder.length() > 128) {
throwIfTagIsNotFound(contentBuilder.toString());
hasTag = true;
}
boolean foundInvalidSubscription = false;
final List<SubscriptionItem> subscriptionItems = new ArrayList<>();
for (final Object subscriptionObject : subscriptions) {
if (!(subscriptionObject instanceof JsonObject)) {
foundInvalidSubscription = true;
continue;
}
} catch (InvalidSourceException e) {
throw e;
} catch (Throwable e) {
throw new InvalidSourceException(e);
} finally {
try {
inputStream.close();
} catch (IOException ignored) {
final JsonObject subscription = ((JsonObject) subscriptionObject).getObject("snippet");
final String id = subscription.getObject("resourceId").getString("channelId", "");
if (id.length() != 24) { // e.g. UCsXVk37bltHxD1rDPwtNM8Q
foundInvalidSubscription = true;
continue;
}
subscriptionItems.add(new SubscriptionItem(service.getServiceId(),
BASE_CHANNEL_URL + id, subscription.getString("title", "")));
}
final String fileContent = contentBuilder.toString().trim();
if (fileContent.isEmpty()) {
throw new InvalidSourceException("Empty input stream");
if (foundInvalidSubscription && subscriptionItems.isEmpty()) {
throw new InvalidSourceException("Found only invalid channel ids");
}
if (!hasTag) {
throwIfTagIsNotFound(fileContent);
}
return fileContent;
return subscriptionItems;
}
}

35
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java
View file

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import java.util.regex.Pattern;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@ -32,6 +33,9 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory();
private static final Pattern excludedSegments =
Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site");
public static YoutubeChannelLinkHandlerFactory getInstance() {
return instance;
}
@ -48,11 +52,22 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
public String getUrl(String id, List<String> contentFilters, String searchFilter) {
return "https://www.youtube.com/" + id;
}
/**
* Returns true if path conform to
* custom short channel URLs like youtube.com/yourcustomname
*
* @param splitPath path segments array
* @return true - if value conform to short channel URL, false - not
*/
private boolean isCustomShortChannelUrl(final String[] splitPath) {
return splitPath.length == 1 && !excludedSegments.matcher(splitPath[0]).matches();
}
@Override
public String getId(String url) throws ParsingException {
try {
URL urlObj = Utils.stringToURL(url);
final URL urlObj = Utils.stringToURL(url);
String path = urlObj.getPath();
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) ||
@ -60,15 +75,21 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
throw new ParsingException("the URL given is not a Youtube-URL");
}
if (!path.startsWith("/user/") && !path.startsWith("/channel/") && !path.startsWith("/c/")) {
// remove leading "/"
path = path.substring(1);
String[] splitPath = path.split("/");
// Handle custom short channel URLs like youtube.com/yourcustomname
if (isCustomShortChannelUrl(splitPath)) {
path = "c/" + path;
splitPath = path.split("/");
}
if (!path.startsWith("user/") && !path.startsWith("channel/") && !path.startsWith("c/")) {
throw new ParsingException("the URL given is neither a channel nor an user");
}
// remove leading "/"
path = path.substring(1);
String[] splitPath = path.split("/");
String id = splitPath[1];
final String id = splitPath[1];
if (id == null || !id.matches("[A-Za-z0-9_-]+")) {
throw new ParsingException("The given id is not a Youtube-Video-ID");

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java
View file

@ -1,10 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
@ -17,15 +14,6 @@ public class YoutubeCommentsLinkHandlerFactory extends ListLinkHandlerFactory {
return instance;
}
@Override
public ListLinkHandler fromUrl(String url) throws ParsingException {
if (url.startsWith(BASE_YOUTUBE_INTENT_URL)){
return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
} else {
return super.fromUrl(url);
}
}
@Override
public String getUrl(String id) {
return "https://m.youtube.com/watch?v=" + id;

39
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View file

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
@ -12,8 +11,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.BASE_YOUTUBE_INTENT_URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
* Created by Christian Schabesberger on 02.02.16.
@ -37,6 +36,7 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("([a-zA-Z0-9_-]{11})");
private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
private YoutubeStreamLinkHandlerFactory() {
@ -46,27 +46,24 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return instance;
}
private static boolean isId(@Nullable String id) {
return id != null && id.matches("[a-zA-Z0-9_-]{11}");
@Nullable
private static String extractId(@Nullable final String id) {
if (id != null) {
final Matcher m = YOUTUBE_VIDEO_ID_REGEX_PATTERN.matcher(id);
return m.find() ? m.group(1) : null;
}
return null;
}
private static String assertIsId(@Nullable String id) throws ParsingException {
if (isId(id)) {
return id;
private static String assertIsId(@Nullable final String id) throws ParsingException {
final String extractedId = extractId(id);
if (extractedId != null) {
return extractedId;
} else {
throw new ParsingException("The given string is not a Youtube-Video-ID");
}
}
@Override
public LinkHandler fromUrl(String url) throws ParsingException {
if (url.startsWith(BASE_YOUTUBE_INTENT_URL)) {
return super.fromUrl(url, BASE_YOUTUBE_INTENT_URL);
} else {
return super.fromUrl(url);
}
}
@Override
public String getUrl(String id) {
return "https://www.youtube.com/watch?v=" + id;
@ -81,9 +78,9 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
if (scheme != null && (scheme.equals("vnd.youtube") || scheme.equals("vnd.youtube.launch"))) {
String schemeSpecificPart = uri.getSchemeSpecificPart();
if (schemeSpecificPart.startsWith("//")) {
final String possiblyId = schemeSpecificPart.substring(2);
if (isId(possiblyId)) {
return possiblyId;
final String extractedId = extractId(schemeSpecificPart.substring(2));
if (extractedId != null) {
return extractedId;
}
urlString = "https:" + schemeSpecificPart;
@ -153,7 +150,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(viewQueryValue);
}
if (path.startsWith("embed/")) {
if (path.startsWith("embed/") || path.startsWith("shorts/")) {
String id = path.split("/")[1];
return assertIsId(id);

1
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View file

@ -321,6 +321,7 @@ public abstract class StreamExtractor extends Extractor {
* @throws IOException
* @throws ExtractionException
*/
@Nullable
public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
/**

6
extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java
View file

@ -8,7 +8,6 @@ import java.util.Locale;
public class SubtitlesStream extends Stream implements Serializable {
private final MediaFormat format;
private final Locale locale;
private final String url;
private final boolean autoGenerated;
private final String code;
@ -34,7 +33,6 @@ public class SubtitlesStream extends Stream implements Serializable {
}
this.code = languageCode;
this.format = format;
this.url = url;
this.autoGenerated = autoGenerated;
}
@ -42,10 +40,6 @@ public class SubtitlesStream extends Stream implements Serializable {
return format.suffix;
}
public String getURL() {
return url;
}
public boolean isAutoGenerated() {
return autoGenerated;
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/subscription/SubscriptionExtractor.java
View file

@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
@ -71,8 +72,9 @@ public abstract class SubscriptionExtractor {
*
* @throws InvalidSourceException when the content read from the InputStream is invalid and can not be parsed
*/
@SuppressWarnings("RedundantThrows")
public List<SubscriptionItem> fromInputStream(InputStream contentInputStream) throws IOException, ExtractionException {
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName() + " doesn't support extracting from an InputStream");
public List<SubscriptionItem> fromInputStream(@Nonnull final InputStream contentInputStream)
throws ExtractionException {
throw new UnsupportedOperationException("Service " + service.getServiceInfo().getName()
+ " doesn't support extracting from an InputStream");
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java
View file

@ -15,16 +15,14 @@ public class DonationLinkHelper {
AMAZON,
}
public static DonationService getDonatoinServiceByLink(String link) throws MalformedURLException {
public static DonationService getDonationServiceByLink(String link) throws MalformedURLException {
URL url = new URL(fixLink(link));
switch (url.getHost()) {
case "www.patreon.com":
return DonationService.PATREON;
case "patreon.com":
return DonationService.PATREON;
case "paypal.me":
return DonationService.PAYPAL;
case "www.paypal.me":
case "paypal.me":
return DonationService.PAYPAL;
default:
return DonationService.NO_DONATION;

35
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View file

@ -181,14 +181,39 @@ public class Utils {
return s;
}
public static String getBaseUrl(String url) throws ParsingException {
URL uri;
public static String getBaseUrl(final String url) throws ParsingException {
try {
uri = stringToURL(url);
} catch (MalformedURLException e) {
final URL uri = stringToURL(url);
return uri.getProtocol() + "://" + uri.getAuthority();
} catch (final MalformedURLException e) {
final String message = e.getMessage();
if (message.startsWith("unknown protocol: ")) {
// return just the protocol (e.g. vnd.youtube)
return message.substring("unknown protocol: ".length());
}
throw new ParsingException("Malformed url: " + url, e);
}
return uri.getProtocol() + "://" + uri.getAuthority();
}
/**
* If the provided url is a Google search redirect, then the actual url is extracted from the
* {@code url=} query value and returned, otherwise the original url is returned.
* @param url the url which can possibly be a Google search redirect
* @return an url with no Google search redirects
*/
public static String followGoogleRedirectIfNeeded(final String url) {
// if the url is a redirect from a Google search, extract the actual url
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
}
} catch (final Exception ignored) {
}
// url is not a google search redirect
return url;
}
public static boolean isNullOrEmpty(final String str) {

94
extractor/src/test/java/org/schabi/newpipe/FileUtils.java Normal file
View file

@ -0,0 +1,94 @@
package org.schabi.newpipe;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
/**
* Util class to write file to disk
* <p>
* Can be used to debug and test, for example writing a service's JSON response
* (especially useful if the response provided by the service is not documented)
*/
public class FileUtils {
public static void createFile(String path, JsonObject content) throws IOException {
createFile(path, jsonObjToString(content));
}
public static void createFile(String path, JsonArray array) throws IOException {
createFile(path, jsonArrayToString(array));
}
/**
* Create a file given a path and its content. Create subdirectories if needed
*
* @param path the path to write the file, including the filename (and its extension)
* @param content the content to write
* @throws IOException
*/
public static void createFile(final String path, final String content) throws IOException {
final String[] dirs = path.split("/");
if (dirs.length > 1) {
String pathWithoutFileName = path.replace(dirs[dirs.length - 1], "");
if (!Files.exists(Paths.get(pathWithoutFileName))) { //create dirs if they don't exist
if (!new File(pathWithoutFileName).mkdirs()) {
throw new IOException("An error occurred while creating directories");
}
}
}
writeFile(path, content);
}
/**
* Write a file to disk
*
* @param filename the file name (and its extension if wanted)
* @param content the content to write
* @throws IOException
*/
private static void writeFile(final String filename, final String content) throws IOException {
final BufferedWriter writer = new BufferedWriter(new FileWriter(filename));
writer.write(content);
writer.flush();
writer.close();
}
/**
* Resolves the test resource file based on its filename. Looks in
* {@code extractor/src/test/resources/} and {@code src/test/resources/}
* @param filename the resource filename
* @return the resource file
*/
public static File resolveTestResource(final String filename) {
final File file = new File("extractor/src/test/resources/" + filename);
if (file.exists()) {
return file;
} else {
return new File("src/test/resources/" + filename);
}
}
/**
* Convert a JSON object to String
* toString() does not produce a valid JSON string
*/
public static String jsonObjToString(JsonObject object) {
return JsonWriter.string(object);
}
/**
* Convert a JSON array to String
* toString() does not produce a valid JSON string
*/
public static String jsonArrayToString(JsonArray array) {
return JsonWriter.string(array);
}
}

33
extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java
View file

@ -1,12 +1,19 @@
package org.schabi.newpipe.extractor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class ExtractorAsserts {
public static void assertEmptyErrors(String message, List<Throwable> errors) {
@ -56,4 +63,22 @@ public class ExtractorAsserts {
assertTrue(message, stringToCheck.isEmpty());
}
}
public static void assertAtLeast(long expected, long actual) {
assertTrue(actual + " is not at least " + expected, actual >= expected);
}
// this assumes that sorting a and b in-place is not an issue, so it's only intended for tests
public static void assertEqualsOrderIndependent(List<String> expected, List<String> actual) {
if (expected == null) {
assertNull(actual);
return;
} else {
assertNotNull(actual);
}
Collections.sort(expected);
Collections.sort(actual);
assertEquals(expected, actual);
}
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java
View file

@ -6,6 +6,7 @@ import java.util.HashSet;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class NewPipeTest {
@ -39,8 +40,10 @@ public class NewPipeTest {
assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube);
assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube);
assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube);
assertEquals(getServiceByUrl("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"), YouTube);
assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube);
assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud);
assertEquals(getServiceByUrl("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="), SoundCloud);
}
@Test

35
extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java Normal file
View file

@ -0,0 +1,35 @@
package org.schabi.newpipe.extractor.services;
public interface BaseStreamExtractorTest extends BaseExtractorTest {
void testStreamType() throws Exception;
void testUploaderName() throws Exception;
void testUploaderUrl() throws Exception;
void testUploaderAvatarUrl() throws Exception;
void testSubChannelName() throws Exception;
void testSubChannelUrl() throws Exception;
void testSubChannelAvatarUrl() throws Exception;
void testThumbnailUrl() throws Exception;
void testDescription() throws Exception;
void testLength() throws Exception;
void testTimestamp() throws Exception;
void testViewCount() throws Exception;
void testUploadDate() throws Exception;
void testTextualUploadDate() throws Exception;
void testLikeCount() throws Exception;
void testDislikeCount() throws Exception;
void testRelatedStreams() throws Exception;
void testAgeLimit() throws Exception;
void testErrorMessage() throws Exception;
void testAudioStreams() throws Exception;
void testVideoStreams() throws Exception;
void testSubtitles() throws Exception;
void testGetDashMpdUrl() throws Exception;
void testFrames() throws Exception;
void testHost() throws Exception;
void testPrivacy() throws Exception;
void testCategory() throws Exception;
void testLicence() throws Exception;
void testLanguageInfo() throws Exception;
void testTags() throws Exception;
void testSupportInfo() throws Exception;
}

382
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java Normal file
View file

@ -0,0 +1,382 @@
package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nullable;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertAtLeast;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEqualsOrderIndependent;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
/**
* Test for {@link StreamExtractor}
*/
public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<StreamExtractor>
implements BaseStreamExtractorTest {
public abstract StreamType expectedStreamType();
public abstract String expectedUploaderName();
public abstract String expectedUploaderUrl();
public String expectedSubChannelName() { return ""; } // default: there is no subchannel
public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
public abstract List<String> expectedDescriptionContains(); // e.g. for full links
public abstract long expectedLength();
public long expectedTimestamp() { return 0; } // default: there is no timestamp
public abstract long expectedViewCountAtLeast();
@Nullable public abstract String expectedUploadDate(); // format: "yyyy-MM-dd HH:mm:ss.SSS"
@Nullable public abstract String expectedTextualUploadDate();
public abstract long expectedLikeCountAtLeast(); // return -1 if ratings are disabled
public abstract long expectedDislikeCountAtLeast(); // return -1 if ratings are disabled
public boolean expectedHasRelatedStreams() { return true; } // default: there are related videos
public int expectedAgeLimit() { return StreamExtractor.NO_AGE_LIMIT; } // default: no limit
@Nullable public String expectedErrorMessage() { return null; } // default: no error message
public boolean expectedHasVideoStreams() { return true; } // default: there are video streams
public boolean expectedHasAudioStreams() { return true; } // default: there are audio streams
public boolean expectedHasSubtitles() { return true; } // default: there are subtitles streams
@Nullable public String expectedDashMpdUrlContains() { return null; } // default: no dash mpd
public boolean expectedHasFrames() { return true; } // default: there are frames
public String expectedHost() { return ""; } // default: no host for centralized platforms
public String expectedPrivacy() { return ""; } // default: no privacy policy available
public String expectedCategory() { return ""; } // default: no category
public String expectedLicence() { return ""; } // default: no licence
public Locale expectedLanguageInfo() { return null; } // default: no language info available
public List<String> expectedTags() { return Collections.emptyList(); } // default: no tags
public String expectedSupportInfo() { return ""; } // default: no support info available
@Test
@Override
public void testStreamType() throws Exception {
assertEquals(expectedStreamType(), extractor().getStreamType());
}
@Test
@Override
public void testUploaderName() throws Exception {
assertEquals(expectedUploaderName(), extractor().getUploaderName());
}
@Test
@Override
public void testUploaderUrl() throws Exception {
final String uploaderUrl = extractor().getUploaderUrl();
assertIsSecureUrl(uploaderUrl);
assertEquals(expectedUploaderUrl(), uploaderUrl);
}
@Test
@Override
public void testUploaderAvatarUrl() throws Exception {
assertIsSecureUrl(extractor().getUploaderAvatarUrl());
}
@Test
@Override
public void testSubChannelName() throws Exception {
assertEquals(expectedSubChannelName(), extractor().getSubChannelName());
}
@Test
@Override
public void testSubChannelUrl() throws Exception {
final String subChannelUrl = extractor().getSubChannelUrl();
assertEquals(expectedSubChannelUrl(), subChannelUrl);
if (!expectedSubChannelUrl().isEmpty()) {
// this stream has a subchannel
assertIsSecureUrl(subChannelUrl);
}
}
@Test
@Override
public void testSubChannelAvatarUrl() throws Exception {
if (expectedSubChannelName().isEmpty() && expectedSubChannelUrl().isEmpty()) {
// this stream has no subchannel
assertEquals("", extractor().getSubChannelAvatarUrl());
} else {
// this stream has a subchannel
assertIsSecureUrl(extractor().getSubChannelAvatarUrl());
}
}
@Test
@Override
public void testThumbnailUrl() throws Exception {
assertIsSecureUrl(extractor().getThumbnailUrl());
}
@Test
@Override
public void testDescription() throws Exception {
final Description description = extractor().getDescription();
assertNotNull(description);
assertFalse("description is empty", description.getContent().isEmpty());
for (final String s : expectedDescriptionContains()) {
assertThat(description.getContent(), containsString(s));
}
}
@Test
@Override
public void testLength() throws Exception {
assertEquals(expectedLength(), extractor().getLength());
}
@Test
@Override
public void testTimestamp() throws Exception {
assertEquals(expectedTimestamp(), extractor().getTimeStamp());
}
@Test
@Override
public void testViewCount() throws Exception {
assertAtLeast(expectedViewCountAtLeast(), extractor().getViewCount());
}
@Test
@Override
public void testUploadDate() throws Exception {
final DateWrapper dateWrapper = extractor().getUploadDate();
if (expectedUploadDate() == null) {
assertNull(dateWrapper);
} else {
assertNotNull(dateWrapper);
final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(),
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"));
final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime();
assertEquals(expectedDateTime, actualDateTime);
}
}
@Test
@Override
public void testTextualUploadDate() throws Exception {
assertEquals(expectedTextualUploadDate(), extractor().getTextualUploadDate());
}
@Test
@Override
public void testLikeCount() throws Exception {
if (expectedLikeCountAtLeast() == -1) {
assertEquals(-1, extractor().getLikeCount());
} else {
assertAtLeast(expectedLikeCountAtLeast(), extractor().getLikeCount());
}
}
@Test
@Override
public void testDislikeCount() throws Exception {
if (expectedDislikeCountAtLeast() == -1) {
assertEquals(-1, extractor().getDislikeCount());
} else {
assertAtLeast(expectedDislikeCountAtLeast(), extractor().getDislikeCount());
}
}
@Test
@Override
public void testRelatedStreams() throws Exception {
final StreamInfoItemsCollector relatedStreams = extractor().getRelatedStreams();
if (expectedHasRelatedStreams()) {
assertNotNull(relatedStreams);
defaultTestListOfItems(extractor().getService(), relatedStreams.getItems(),
relatedStreams.getErrors());
} else {
assertNull(relatedStreams);
}
}
@Test
@Override
public void testAgeLimit() throws Exception {
assertEquals(expectedAgeLimit(), extractor().getAgeLimit());
}
@Test
@Override
public void testErrorMessage() throws Exception {
assertEquals(expectedErrorMessage(), extractor().getErrorMessage());
}
@Test
@Override
public void testVideoStreams() throws Exception {
final List<VideoStream> videoStreams = extractor().getVideoStreams();
final List<VideoStream> videoOnlyStreams = extractor().getVideoOnlyStreams();
assertNotNull(videoStreams);
assertNotNull(videoOnlyStreams);
videoStreams.addAll(videoOnlyStreams);
if (expectedHasVideoStreams()) {
assertFalse(videoStreams.isEmpty());
for (final VideoStream stream : videoStreams) {
assertIsSecureUrl(stream.getUrl());
assertFalse(stream.getResolution().isEmpty());
final int formatId = stream.getFormatId();
// see MediaFormat: video stream formats range from 0 to 0x100
assertTrue("format id does not fit a video stream: " + formatId,
0 <= formatId && formatId < 0x100);
}
} else {
assertTrue(videoStreams.isEmpty());
}
}
@Test
@Override
public void testAudioStreams() throws Exception {
final List<AudioStream> audioStreams = extractor().getAudioStreams();
assertNotNull(audioStreams);
if (expectedHasAudioStreams()) {
assertFalse(audioStreams.isEmpty());
for (final AudioStream stream : audioStreams) {
assertIsSecureUrl(stream.getUrl());
final int formatId = stream.getFormatId();
// see MediaFormat: video stream formats range from 0x100 to 0x1000
assertTrue("format id does not fit an audio stream: " + formatId,
0x100 <= formatId && formatId < 0x1000);
}
} else {
assertTrue(audioStreams.isEmpty());
}
}
@Test
@Override
public void testSubtitles() throws Exception {
final List<SubtitlesStream> subtitles = extractor().getSubtitlesDefault();
assertNotNull(subtitles);
if (expectedHasSubtitles()) {
assertFalse(subtitles.isEmpty());
for (final SubtitlesStream stream : subtitles) {
assertIsSecureUrl(stream.getUrl());
final int formatId = stream.getFormatId();
// see MediaFormat: video stream formats range from 0x1000 to 0x10000
assertTrue("format id does not fit a subtitles stream: " + formatId,
0x1000 <= formatId && formatId < 0x10000);
}
} else {
assertTrue(subtitles.isEmpty());
final MediaFormat[] formats = {MediaFormat.VTT, MediaFormat.TTML, MediaFormat.SRT,
MediaFormat.TRANSCRIPT1, MediaFormat.TRANSCRIPT2, MediaFormat.TRANSCRIPT3};
for (final MediaFormat format : formats) {
final List<SubtitlesStream> formatSubtitles = extractor().getSubtitles(format);
assertNotNull(formatSubtitles);
assertTrue(formatSubtitles.isEmpty());
}
}
}
@Override
public void testGetDashMpdUrl() throws Exception {
final String dashMpdUrl = extractor().getDashMpdUrl();
if (expectedDashMpdUrlContains() == null) {
assertNotNull(dashMpdUrl);
assertTrue(dashMpdUrl.isEmpty());
} else {
assertIsSecureUrl(dashMpdUrl);
assertThat(extractor().getDashMpdUrl(), containsString(expectedDashMpdUrlContains()));
}
}
@Test
@Override
public void testFrames() throws Exception {
final List<Frameset> frames = extractor().getFrames();
assertNotNull(frames);
if (expectedHasFrames()) {
assertFalse(frames.isEmpty());
for (final Frameset f : frames) {
for (final String url : f.getUrls()) {
assertIsValidUrl(url);
assertIsSecureUrl(url);
}
}
} else {
assertTrue(frames.isEmpty());
}
}
@Test
@Override
public void testHost() throws Exception {
assertEquals(expectedHost(), extractor().getHost());
}
@Test
@Override
public void testPrivacy() throws Exception {
assertEquals(expectedPrivacy(), extractor().getPrivacy());
}
@Test
@Override
public void testCategory() throws Exception {
assertEquals(expectedCategory(), extractor().getCategory());
}
@Test
@Override
public void testLicence() throws Exception {
assertEquals(expectedLicence(), extractor().getLicence());
}
@Test
@Override
public void testLanguageInfo() throws Exception {
assertEquals(expectedLanguageInfo(), extractor().getLanguageInfo());
}
@Test
@Override
public void testTags() throws Exception {
assertEqualsOrderIndependent(expectedTags(), extractor().getTags());
}
@Test
@Override
public void testSupportInfo() throws Exception {
assertEquals(expectedSupportInfo(), extractor().getSupportInfo());
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View file

@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -42,7 +41,7 @@ public final class DefaultTests {
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
final String uploaderUrl = streamInfoItem.getUploaderUrl();
if (!isNullOrEmpty(uploaderUrl)) {
assertIsSecureUrl(uploaderUrl);
@ -54,7 +53,6 @@ public final class DefaultTests {
if (!isNullOrEmpty(streamInfoItem.getTextualUploadDate())) {
final DateWrapper uploadDate = streamInfoItem.getUploadDate();
assertNotNull("No parsed upload date", uploadDate);
assertTrue("Upload date not in the past", uploadDate.date().before(Calendar.getInstance()));
}
} else if (item instanceof ChannelInfoItem) {

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java
View file

@ -31,7 +31,7 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetUrl() throws Exception {
assertEquals("https://media.ccc.de/public/conferences/froscon2017", extractor.getUrl());
assertEquals("https://media.ccc.de/c/froscon2017", extractor.getUrl());
}
@Test
@ -67,7 +67,7 @@ public class MediaCCCConferenceExtractorTest {
@Test
public void testGetUrl() throws Exception {
assertEquals("https://media.ccc.de/public/conferences/oscal19", extractor.getUrl());
assertEquals("https://media.ccc.de/c/oscal19", extractor.getUrl());
}
@Test

42
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceLinkHandlerFactoryTest.java Normal file
View file

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import static org.junit.Assert.assertEquals;
public class MediaCCCConferenceLinkHandlerFactoryTest {
private static MediaCCCConferenceLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = new MediaCCCConferenceLinkHandlerFactory();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void getId() throws ParsingException {
assertEquals("jh20",
linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getId());
assertEquals("jh20",
linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getId());
assertEquals("jh20",
linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getId());
}
@Test
public void getUrl() throws ParsingException {
assertEquals("https://media.ccc.de/c/jh20",
linkHandler.fromUrl("https://media.ccc.de/c/jh20#278").getUrl());
assertEquals("https://media.ccc.de/c/jh20",
linkHandler.fromUrl("https://media.ccc.de/b/jh20?a=b").getUrl());
assertEquals("https://media.ccc.de/c/jh20",
linkHandler.fromUrl("https://api.media.ccc.de/public/conferences/jh20&a=b&b=c").getUrl());
assertEquals("https://media.ccc.de/c/jh20",
linkHandler.fromId("jh20").getUrl());
}
}

218
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View file

@ -1,204 +1,152 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Arrays;
import java.util.List;
import static java.util.Objects.requireNonNull;
import javax.annotation.Nullable;
import static junit.framework.TestCase.assertEquals;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
/**
* Test {@link MediaCCCStreamExtractor}
*/
public class MediaCCCStreamExtractorTest {
public static class Gpn18Tmux {
private static MediaCCCStreamExtractor extractor;
private static final String BASE_URL = "https://media.ccc.de/v/";
public static class Gpn18Tmux extends DefaultStreamExtractorTest {
private static final String ID = "gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUpClass() throws Exception {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht");
extractor = MediaCCC.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testServiceId() throws Exception {
assertEquals(2, extractor.getServiceId());
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return "tmux - Warum ein schwarzes Fenster am Bildschirm reicht"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testName() throws Exception {
assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName());
}
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "gpn18"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/gpn18"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("SSH-Sessions", "\"Terminal Multiplexer\""); }
@Override public long expectedLength() { return 3097; }
@Override public long expectedViewCountAtLeast() { return 2380; }
@Nullable @Override public String expectedUploadDate() { return "2018-05-11 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2018-05-11T02:00:00.000+02:00"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
@Override
@Test
public void testId() throws Exception {
assertEquals("gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getId());
}
@Test
public void testUrl() throws Exception {
assertIsSecureUrl(extractor.getUrl());
assertEquals("https://media.ccc.de/public/events/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws Exception {
assertIsSecureUrl(extractor.getOriginalUrl());
assertEquals("https://media.ccc.de/v/gpn18-105-tmux-warum-ein-schwarzes-fenster-am-bildschirm-reicht", extractor.getOriginalUrl());
}
@Test
public void testThumbnail() throws Exception {
assertIsSecureUrl(extractor.getThumbnailUrl());
public void testThumbnailUrl() throws Exception {
super.testThumbnailUrl();
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl());
}
@Test
public void testUploaderName() throws Exception {
assertEquals("gpn18", extractor.getUploaderName());
}
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
}
@Override
@Test
public void testUploaderAvatarUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
super.testUploaderAvatarUrl();
assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl());
}
@Override
@Test
public void testVideoStreams() throws Exception {
List<VideoStream> videoStreamList = extractor.getVideoStreams();
assertEquals(4, videoStreamList.size());
for (VideoStream stream : videoStreamList) {
assertIsSecureUrl(stream.getUrl());
}
super.testVideoStreams();
assertEquals(4, extractor.getVideoStreams().size());
}
@Override
@Test
public void testAudioStreams() throws Exception {
List<AudioStream> audioStreamList = extractor.getAudioStreams();
assertEquals(2, audioStreamList.size());
for (AudioStream stream : audioStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2018-05-11"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
super.testAudioStreams();
assertEquals(2, extractor.getAudioStreams().size());
}
}
public static class _36c3PrivacyMessaging {
private static MediaCCCStreamExtractor extractor;
public static class _36c3PrivacyMessaging extends DefaultStreamExtractorTest {
private static final String ID = "36c3-10565-what_s_left_for_private_messaging";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUpClass() throws Exception {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (MediaCCCStreamExtractor) MediaCCC.getStreamExtractor("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging");
extractor = MediaCCC.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testName() throws Exception {
assertEquals("What's left for private messaging?", extractor.getName());
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return MediaCCC; }
@Override public String expectedName() { return "What's left for private messaging?"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testId() throws Exception {
assertEquals("36c3-10565-what_s_left_for_private_messaging", extractor.getId());
}
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "36c3"; }
@Override public String expectedUploaderUrl() { return "https://media.ccc.de/c/36c3"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("WhatsApp", "Signal"); }
@Override public long expectedLength() { return 3603; }
@Override public long expectedViewCountAtLeast() { return 2380; }
@Nullable @Override public String expectedUploadDate() { return "2020-01-11 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2020-01-11T01:00:00.000+01:00"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasRelatedStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("36c3", "10565", "2019", "Security", "Main"); }
@Override
@Test
public void testUrl() throws Exception {
assertIsSecureUrl(extractor.getUrl());
assertEquals("https://media.ccc.de/public/events/36c3-10565-what_s_left_for_private_messaging", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws Exception {
assertIsSecureUrl(extractor.getOriginalUrl());
assertEquals("https://media.ccc.de/v/36c3-10565-what_s_left_for_private_messaging", extractor.getOriginalUrl());
}
@Test
public void testThumbnail() throws Exception {
assertIsSecureUrl(extractor.getThumbnailUrl());
public void testThumbnailUrl() throws Exception {
super.testThumbnailUrl();
assertEquals("https://static.media.ccc.de/media/congress/2019/10565-hd.jpg", extractor.getThumbnailUrl());
}
@Test
public void testUploaderName() throws Exception {
assertEquals("36c3", extractor.getUploaderName());
}
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://media.ccc.de/public/conferences/36c3", extractor.getUploaderUrl());
}
@Override
@Test
public void testUploaderAvatarUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
super.testUploaderAvatarUrl();
assertEquals("https://static.media.ccc.de/media/congress/2019/logo.png", extractor.getUploaderAvatarUrl());
}
@Override
@Test
public void testVideoStreams() throws Exception {
List<VideoStream> videoStreamList = extractor.getVideoStreams();
assertEquals(8, videoStreamList.size());
for (VideoStream stream : videoStreamList) {
assertIsSecureUrl(stream.getUrl());
}
super.testVideoStreams();
assertEquals(8, extractor.getVideoStreams().size());
}
@Override
@Test
public void testAudioStreams() throws Exception {
List<AudioStream> audioStreamList = extractor.getAudioStreams();
assertEquals(2, audioStreamList.size());
for (AudioStream stream : audioStreamList) {
assertIsSecureUrl(stream.getUrl());
}
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2020-01-11T01:00:00.000+01:00", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-11"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
super.testAudioStreams();
assertEquals(2, extractor.getAudioStreams().size());
}
}
}

42
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamLinkHandlerFactoryTest.java Normal file
View file

@ -0,0 +1,42 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import static org.junit.Assert.assertEquals;
public class MediaCCCStreamLinkHandlerFactoryTest {
private static MediaCCCStreamLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
linkHandler = new MediaCCCStreamLinkHandlerFactory();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void getId() throws ParsingException {
assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getId());
assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?a=b").getId());
assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020#3").getId());
assertEquals("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020&a=b").getId());
}
@Test
public void getUrl() throws ParsingException {
assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl());
assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromUrl("https://api.media.ccc.de/public/events/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020?b=a&a=b").getUrl());
assertEquals("https://media.ccc.de/v/jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020",
linkHandler.fromId("jhremote20-3001-abschlusspraesentation_jugend_hackt_remote_2020").getUrl());
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java
View file

@ -28,7 +28,7 @@ public class PeertubeAccountExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/kde");
.getChannelExtractor("https://peertube.mastodon.host/accounts/kde");
extractor.fetchPage();
}
@ -53,7 +53,7 @@ public class PeertubeAccountExtractorTest {
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kde", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getUrl());
}
@Test
@ -89,10 +89,9 @@ public class PeertubeAccountExtractorTest {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
public void testBannerUrl() {
assertNull(extractor.getBannerUrl());
}
@Test
@ -115,7 +114,7 @@ public class PeertubeAccountExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/booteille");
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/booteille");
extractor.fetchPage();
}
@ -150,12 +149,12 @@ public class PeertubeAccountExtractorTest {
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////

13
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelExtractorTest.java
View file

@ -28,7 +28,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
.getChannelExtractor("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
extractor.fetchPage();
}
@ -53,7 +53,7 @@ public class PeertubeChannelExtractorTest {
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
}
@Test
@ -104,10 +104,9 @@ public class PeertubeChannelExtractorTest {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
assertNull(extractor.getBannerUrl());
}
@Test
@ -130,7 +129,7 @@ public class PeertubeChannelExtractorTest {
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
.getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
extractor.fetchPage();
}
@ -165,12 +164,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////

34
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeChannelLinkHandlerFactoryTest.java
View file

@ -28,20 +28,36 @@ public class PeertubeChannelLinkHandlerFactoryTest {
@Test
public void acceptUrlTest() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa"));
}
@Test
public void getIdFromUrl() throws ParsingException {
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("accounts/kranti@videos.squat.net", linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
public void getId() throws ParsingException {
assertEquals("accounts/kranti@videos.squat.net",
linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net").getId());
assertEquals("accounts/kranti@videos.squat.net",
linkHandler.fromUrl("https://peertube.mastodon.host/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa/videos").getId());
assertEquals("accounts/kranti@videos.squat.net",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net").getId());
assertEquals("accounts/kranti@videos.squat.net",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net/videos").getId());
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getId());
}
@Test
public void getUrlFromId() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/kranti@videos.squat.net", linkHandler.fromId("kranti@videos.squat.net").getUrl());
public void getUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromId("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net",
linkHandler.fromId("accounts/kranti@videos.squat.net").getUrl());
assertEquals("https://peertube.mastodon.host/accounts/kranti@videos.squat.net",
linkHandler.fromId("kranti@videos.squat.net").getUrl());
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa").getUrl());
}
}

180
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View file

@ -1,180 +0,0 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Locale;
import java.util.TimeZone;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
/**
* Test for {@link StreamExtractor}
*/
public class PeertubeStreamExtractorDefaultTest {
private static PeertubeStreamExtractor extractor;
private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/";
private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
extractor.fetchPage();
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
instance.setTime(sdf.parse("2018-10-01T10:52:46.396Z"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetTitle() throws ParsingException {
assertEquals("What is PeerTube?", extractor.getName());
}
@Test
public void testGetLargeDescription() throws ParsingException {
assertEquals(expectedLargeDescription, extractor.getDescription().getContent());
}
@Test
public void testGetEmptyDescription() throws Exception {
PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
extractorEmpty.fetchPage();
assertEquals("", extractorEmpty.getDescription().getContent());
}
@Test
public void testGetSmallDescription() throws Exception {
PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
extractorSmall.fetchPage();
assertEquals(expectedSmallDescription, extractorSmall.getDescription().getContent());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals("Framasoft", extractor.getUploaderName());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetSubChannelName() throws ParsingException {
assertEquals("Les vidéos de Framasoft", extractor.getSubChannelName());
}
@Test
public void testGetSubChannelUrl() throws ParsingException {
assertIsSecureUrl(extractor.getSubChannelUrl());
assertEquals("https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8", extractor.getSubChannelUrl());
}
@Test
public void testGetSubChannelAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getSubChannelAvatarUrl());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(113, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
assertTrue(Long.toString(extractor.getViewCount()),
extractor.getViewCount() > 10);
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
assertFalse(extractor.getVideoStreams().isEmpty());
}
@Test
public void testStreamType() throws ParsingException {
assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM);
}
@Ignore
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetAgeLimit() throws ExtractionException, IOException {
assertEquals(0, extractor.getAgeLimit());
PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://nocensoring.net/videos/embed/dbd8e5e1-c527-49b6-b70c-89101dbb9c08");
ageLimit.fetchPage();
assertEquals(18, ageLimit.getAgeLimit());
}
@Test
public void testGetSupportInformation() throws ExtractionException, IOException {
PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
supportInfoExtractor.fetchPage();
assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
}
@Test
public void testGetLanguageInformation() throws ParsingException {
assertEquals(new Locale("en"), extractor.getLanguageInfo());
}
}

172
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java Normal file
View file

@ -0,0 +1,172 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nullable;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeStreamExtractorTest {
private static final String BASE_URL = "/videos/watch/";
public static class WhatIsPeertube extends DefaultStreamExtractorTest {
private static final String ID = "9c9de5e8-0a1e-484a-b099-e80766180a6d";
private static final String INSTANCE = "https://framatube.org";
private static final int TIMESTAMP_MINUTE = 1;
private static final int TIMESTAMP_SECOND = 21;
private static final String URL = INSTANCE + BASE_URL + ID + "?start=" + TIMESTAMP_MINUTE + "m" + TIMESTAMP_SECOND + "s";
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel (!)
PeerTube.setInstance(new PeertubeInstance(INSTANCE, "FramaTube"));
extractor = PeerTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetLanguageInformation() throws ParsingException {
assertEquals(new Locale("en"), extractor.getLanguageInfo());
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return PeerTube; }
@Override public String expectedName() { return "What is PeerTube?"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Framasoft"; }
@Override public String expectedUploaderUrl() { return "https://framatube.org/accounts/framasoft@framatube.org"; }
@Override public String expectedSubChannelName() { return "Les vidéos de Framasoft"; }
@Override public String expectedSubChannelUrl() { return "https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8"; }
@Override public List<String> expectedDescriptionContains() { // CRLF line ending
return Arrays.asList("**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n"
+ "\r\n"
+ "**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n"
+ "*A decentralized video hosting network, based on free/libre software!*\r\n"
+ "\r\n"
+ "**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n"
+ "*Directed by* Aryeom\r\n"
+ "*Assistant* Jehan\r\n"
+ "**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n"
+ "\r\n"
+ "**Sponsored by** [Framasoft](https://framasoft.org)\r\n"
+ "\r\n"
+ "**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n"
+ "\r\n"
+ "**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n"
+ "\r\n"
+ "**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/");
}
@Override public long expectedLength() { return 113; }
@Override public long expectedTimestamp() { return TIMESTAMP_MINUTE*60 + TIMESTAMP_SECOND; }
@Override public long expectedViewCountAtLeast() { return 38600; }
@Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
@Override public long expectedLikeCountAtLeast() { return 120; }
@Override public long expectedDislikeCountAtLeast() { return 0; }
@Override public boolean expectedHasAudioStreams() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "framatube.org"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Science & Technology"; }
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
}
public static class AgeRestricted extends DefaultStreamExtractorTest {
private static final String ID = "dbd8e5e1-c527-49b6-b70c-89101dbb9c08";
private static final String INSTANCE = "https://nocensoring.net";
private static final String URL = INSTANCE + "/videos/embed/" + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());;
// setting instance might break test when running in parallel (!)
PeerTube.setInstance(new PeertubeInstance(INSTANCE));
extractor = PeerTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return PeerTube; }
@Override public String expectedName() { return "Covid-19 ? [Court-métrage]"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return INSTANCE + BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Résilience humaine"; }
@Override public String expectedUploaderUrl() { return "https://nocensoring.net/accounts/gmt@nocensoring.net"; }
@Override public String expectedSubChannelName() { return "SYSTEM FAILURE Quel à-venir ?"; }
@Override public String expectedSubChannelUrl() { return "https://nocensoring.net/video-channels/systemfailure_quel"; }
@Override public List<String> expectedDescriptionContains() { // LF line ending
return Arrays.asList("2020, le monde est frappé par une pandémie, beaucoup d'humains sont confinés.",
"System Failure Quel à-venir ? - Covid-19 / 2020");
}
@Override public long expectedLength() { return 667; }
@Override public long expectedViewCountAtLeast() { return 138; }
@Nullable @Override public String expectedUploadDate() { return "2020-05-14 17:24:35.580"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2020-05-14T17:24:35.580Z"; }
@Override public long expectedLikeCountAtLeast() { return 1; }
@Override public long expectedDislikeCountAtLeast() { return 0; }
@Override public int expectedAgeLimit() { return 18; }
@Override public boolean expectedHasAudioStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public String expectedHost() { return "nocensoring.net"; }
@Override public String expectedPrivacy() { return "Public"; }
@Override public String expectedCategory() { return "Art"; }
@Override public String expectedLicence() { return "Attribution"; }
@Override public List<String> expectedTags() { return Arrays.asList("Covid-19", "Gérôme-Mary trebor", "Horreur et beauté", "court-métrage", "nue artistique"); }
}
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server"));
}
@Test
public void testGetEmptyDescription() throws Exception {
StreamExtractor extractorEmpty = PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d");
extractorEmpty.fetchPage();
assertEquals("", extractorEmpty.getDescription().getContent());
}
@Test
public void testGetSmallDescription() throws Exception {
StreamExtractor extractorSmall = PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea");
extractorSmall.fetchPage();
assertEquals("https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary", extractorSmall.getDescription().getContent());
}
@Test
public void testGetSupportInformation() throws ExtractionException, IOException {
StreamExtractor supportInfoExtractor = PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37");
supportInfoExtractor.fetchPage();
assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo());
}
}

28
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamLinkHandlerFactoryTest.java
View file

@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStream
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
/**
* Test for {@link PeertubeStreamLinkHandlerFactory}
@ -17,16 +18,34 @@ public class PeertubeStreamLinkHandlerFactoryTest {
private static PeertubeStreamLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() throws Exception {
public static void setUp() {
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
linkHandler = PeertubeStreamLinkHandlerFactory.getInstance();
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test
public void getId() throws Exception {
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60", linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d", linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
assertEquals("9c9de5e8-0a1e-484a-b099-e80766180a6d",
linkHandler.fromUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getId());
assertEquals("986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa").getId());
}
@Test
public void getUrl() throws Exception {
assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromId("986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
assertEquals("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60",
linkHandler.fromUrl("https://peertube.mastodon.host/videos/embed/986aac60-1263-4f73-9ce5-36b18225cb60").getUrl());
}
@ -35,5 +54,6 @@ public class PeertubeStreamLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/embed/9c9de5e8-0a1e-484a-b099-e80766180a6d"));
assertTrue(linkHandler.acceptUrl("https://peertube.mastodon.host/api/v1/videos/watch/986aac60-1263-4f73-9ce5-36b18225cb60?fsdafs=fsafa"));
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeTrendingExtractorTest.java
View file

@ -8,8 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeTrendingExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.*;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
public class PeertubeTrendingExtractorTest {

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractorTest.java
View file

@ -41,7 +41,7 @@ public class SoundcloudChannelExtractorTest {
@Test
public void testName() {
assertEquals("LIL UZI VERT", extractor.getName());
assertEquals("Lil Uzi Vert", extractor.getName());
}
@Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java
View file

@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChartsExtractor;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View file

@ -98,7 +98,7 @@ public class SoundcloudPlaylistExtractorTest {
@Test
public void testUploaderName() {
assertTrue(extractor.getUploaderName().contains("LIL UZI VERT"));
assertTrue(extractor.getUploaderName().contains("Lil Uzi Vert"));
}
@Test

163
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View file

@ -1,163 +0,0 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.TimeZone;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
/**
* Test for {@link StreamExtractor}
*/
public class SoundcloudStreamExtractorDefaultTest {
public static class LilUziVertDoWhatIWant {
private static SoundcloudStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (SoundcloudStreamExtractor) SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69");
assertEquals("69", extractor.getTimeStamp() + "");
}
@Test
public void testGetTitle() throws ParsingException {
assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName());
}
@Test
public void testGetDescription() throws ParsingException {
assertEquals("The Perfect LUV Tape®️", extractor.getDescription().getContent());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertEquals("LIL UZI VERT", extractor.getUploaderName());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(175, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
assertTrue(Long.toString(extractor.getViewCount()),
extractor.getViewCount() > 44227978);
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2016-07-31 18:18:07", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
instance.setTime(sdf.parse("2016/07/31 18:18:07 +0000"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://soundcloud.com/liluzivert", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testStreamType() throws ParsingException {
assertTrue(extractor.getStreamType() == StreamType.AUDIO_STREAM);
}
@Test
public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
}
public static class ContentNotSupported {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}
@Test(expected = ContentNotSupportedException.class)
public void hlsAudioStream() throws Exception {
final StreamExtractor extractor =
SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool");
extractor.fetchPage();
extractor.getAudioStreams();
}
@Test(expected = ContentNotSupportedException.class)
public void bothHlsAndOpusAudioStreams() throws Exception {
final StreamExtractor extractor =
SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker");
extractor.fetchPage();
extractor.getAudioStreams();
}
}
}

60
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java Normal file
View file

@ -0,0 +1,60 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
public class SoundcloudStreamExtractorTest {
public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
private static final String ID = "plays-well-with-others-ep-2-what-do-an-army-of-ants-and-an-online-encyclopedia-have-in-common";
private static final String UPLOADER = "https://soundcloud.com/wearecc";
private static final int TIMESTAMP = 69;
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getStreamExtractor(URL);
extractor.fetchPage();
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return "Plays Well with Others, Ep 2: What Do an Army of Ants and an Online Encyclopedia Have in Common?"; }
@Override public String expectedId() { return "597253485"; }
@Override public String expectedUrlContains() { return UPLOADER + "/" + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; }
@Override public String expectedUploaderName() { return "Creative Commons"; }
@Override public String expectedUploaderUrl() { return UPLOADER; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Stigmergy is a mechanism of indirect coordination",
"All original content in Plays Well with Others is available under a Creative Commons BY license."); }
@Override public long expectedLength() { return 1400; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 27000; }
@Nullable @Override public String expectedUploadDate() { return "2019-03-28 13:36:18.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-03-28 13:36:18"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasVideoStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java
View file

@ -53,6 +53,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
assertEquals("309689103", linkHandler.fromUrl("https://soundcloud.com/liluzivert/15-ysl").getId());
assertEquals("309689082", linkHandler.fromUrl("https://www.soundcloud.com/liluzivert/15-luv-scars-ko").getId());
assertEquals("309689035", linkHandler.fromUrl("http://soundcloud.com/liluzivert/15-boring-shit").getId());
assertEquals("259273264", linkHandler.fromUrl("https://soundcloud.com/liluzivert/ps-qs-produced-by-don-cannon/").getId());
assertEquals("294488599", linkHandler.fromUrl("http://www.soundcloud.com/liluzivert/secure-the-bag-produced-by-glohan-beats").getId());
assertEquals("294488438", linkHandler.fromUrl("HtTpS://sOuNdClOuD.cOm/LiLuZiVeRt/In-O4-pRoDuCeD-bY-dP-bEaTz").getId());
assertEquals("294488147", linkHandler.fromUrl("https://soundcloud.com/liluzivert/fresh-produced-by-zaytoven#t=69").getId());
@ -60,6 +61,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
assertEquals("294487684", linkHandler.fromUrl("https://soundcloud.com/liluzivert/blonde-brigitte-produced-manny-fresh#t=1:9").getId());
assertEquals("294487428", linkHandler.fromUrl("https://soundcloud.com/liluzivert/today-produced-by-c-note#t=1m9s").getId());
assertEquals("294487157", linkHandler.fromUrl("https://soundcloud.com/liluzivert/changed-my-phone-produced-by-c-note#t=1m09s").getId());
assertEquals("44556776", linkHandler.fromUrl("https://soundcloud.com/kechuspider-sets-1/last-days").getId());
}

64
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java Normal file
View file

@ -0,0 +1,64 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class YouTubeCommentsLinkHandlerFactoryTest {
private static YoutubeCommentsLinkHandlerFactory linkHandler;
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
linkHandler = YoutubeCommentsLinkHandlerFactory.getInstance();
}
@Test(expected = IllegalArgumentException.class)
public void getIdWithNullAsUrl() throws ParsingException {
linkHandler.fromId(null);
}
@Test
public void getIdFromYt() throws ParsingException {
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.youtube.com/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://m.youtube.com/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtube.com/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://youtu.be/VM_6n762j6M&t=20").getId());
}
@Test
public void testAcceptUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=VM_6n762j6M&t=20"));
assertTrue(linkHandler.acceptUrl("https://WWW.youtube.com/watch?v=VM_6n762j6M&t=20"));
assertTrue(linkHandler.acceptUrl("https://youtube.com/watch?v=VM_6n762j6M&t=20"));
assertTrue(linkHandler.acceptUrl("https://youtu.be/VM_6n762j6M&t=20"));
}
@Test
public void testDeniesUrl() throws ParsingException {
assertFalse(linkHandler.acceptUrl("https://www.you com/watch?v=VM_6n762j6M"));
assertFalse(linkHandler.acceptUrl("https://com/watch?v=VM_6n762j6M"));
assertFalse(linkHandler.acceptUrl("htt ://com/watch?v=VM_6n762j6M"));
assertFalse(linkHandler.acceptUrl("ftp://www.youtube.com/watch?v=VM_6n762j6M"));
}
@Test
public void getIdFromInvidious() throws ParsingException {
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://www.invidio.us/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://INVIDIO.US/watch?v=VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M").getId());
assertEquals("VM_6n762j6M", linkHandler.fromUrl("https://invidio.us/VM_6n762j6M&t=20").getId());
}
}

184
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View file

@ -432,190 +432,6 @@ public class YoutubeChannelExtractorTest {
}
}
// this channel has no "Subscribe" button
public static class EminemVEVO implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertEquals("EminemVEVO", extractor.getName());
}
@Test
public void testId() throws Exception {
assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws Exception {
final String description = extractor.getDescription();
assertTrue(description, description.contains("Eminem on Vevo"));
}
@Test
public void testAvatarUrl() throws Exception {
String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
}
@Test
public void testBannerUrl() throws Exception {
String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt3"));
}
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws Exception {
// there is no "Subscribe" button
long subscribers = extractor.getSubscriberCount();
assertEquals("Wrong subscriber count", -1, subscribers);
}
}
/**
* Some VEVO channels will redirect to a new page with a new channel id.
* <p>
* Though, it isn't a simple redirect, but a redirect instruction embed in the response itself, this
* test assure that we account for that.
*/
public static class RedirectedChannel implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}
@Test
public void testName() throws Exception {
assertEquals("LordiVEVO", extractor.getName());
}
@Test
public void testId() throws Exception {
assertEquals("UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCITk7Ky4iE5_xISw9IaHqpQ", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}
@Test
public void testMoreRelatedItems() throws Exception {
assertNoMoreItems(extractor);
}
/*//////////////////////////////////////////////////////////////////////////
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testDescription() throws Exception {
assertEmpty(extractor.getDescription());
}
@Test
public void testAvatarUrl() throws Exception {
String avatarUrl = extractor.getAvatarUrl();
assertIsSecureUrl(avatarUrl);
assertTrue(avatarUrl, avatarUrl.contains("yt3"));
}
@Test
public void testBannerUrl() throws Exception {
assertEmpty(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws Exception {
assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UCrxkwepj7-4Wz1wHyfzw-sQ", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws Exception {
assertEquals(-1, extractor.getSubscriberCount());
}
}
public static class RandomChannel implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLinkHandlerFactoryTest.java
View file

@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@ -30,6 +31,8 @@ public class YoutubeChannelLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/c/creatoracademy"));
assertTrue(linkHandler.acceptUrl("https://youtube.com/DIMENSI0N"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
@ -44,6 +47,18 @@ public class YoutubeChannelLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/channel/UClq42foiSgl7sSpLupnugGA/videos?disable_polymer=1"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watchismo"));
// do not accept URLs which are not channels
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100"));
assertFalse(linkHandler.acceptUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
assertFalse(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1d"));
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV"));
assertFalse(linkHandler.acceptUrl("https://www.youtube.com/?app=desktop&persist_app=1"));
assertFalse(linkHandler.acceptUrl("https://m.youtube.com/select_site"));
}
@Test

19
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java
View file

@ -6,12 +6,15 @@ import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.text.SimpleDateFormat;
import java.util.*;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@ -23,7 +26,7 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
@Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true;
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Test
public void testAllSupportedLocalizations() throws Exception {
@ -64,7 +67,7 @@ public class YoutubeChannelLocalizationTest {
+ "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount();
final DateWrapper uploadDate = item.getUploadDate();
if (uploadDate != null) {
String dateAsText = dateFormat.format(uploadDate.date().getTime());
String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime());
debugMessage += "\n:::: " + item.getTextualUploadDate() +
"\n:::: " + dateAsText;
}
@ -107,13 +110,13 @@ public class YoutubeChannelLocalizationTest {
final DateWrapper currentUploadDate = currentItem.getUploadDate();
final String referenceDateString = referenceUploadDate == null ? "null" :
dateFormat.format(referenceUploadDate.date().getTime());
dateTimeFormatter.format(referenceUploadDate.offsetDateTime());
final String currentDateString = currentUploadDate == null ? "null" :
dateFormat.format(currentUploadDate.date().getTime());
dateTimeFormatter.format(currentUploadDate.offsetDateTime());
long difference = -1;
if (referenceUploadDate != null && currentUploadDate != null) {
difference = Math.abs(referenceUploadDate.date().getTimeInMillis() - currentUploadDate.date().getTimeInMillis());
difference = ChronoUnit.MILLIS.between(referenceUploadDate.offsetDateTime(), currentUploadDate.offsetDateTime());
}
final boolean areTimeEquals = difference < 5 * 60 * 1000L;

190
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View file

@ -23,91 +23,133 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
private static YoutubeCommentsExtractor extractorYT;
private static YoutubeCommentsExtractor extractorInvidious;
/**
* Test a "normal" YouTube
*/
public static class Thomas {
private static final String url = "https://www.youtube.com/watch?v=D00Au7k3i6o";
private static YoutubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractorYT = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlYT);
extractorYT.fetchPage();
extractorInvidious = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlInvidious);
}
private static final String commentContent = "sub 4 sub";
@Test
public void testGetComments() throws IOException, ExtractionException {
assertTrue(getCommentsHelper(extractorYT));
assertTrue(getCommentsHelper(extractorInvidious));
}
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
boolean result = findInComments(comments, "s1ck m3m3");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "s1ck m3m3");
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
return result;
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
}
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPage();
@Test
public void testGetComments() throws IOException, ExtractionException {
assertTrue(getCommentsHelper(extractor));
}
return result;
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
boolean result = findInComments(comments, commentContent);
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, commentContent);
}
return result;
}
}
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
assertTrue(getCommentsFromCommentsInfoHelper(url));
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), commentContent);
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), commentContent);
nextPage = moreItems.getNextPage();
}
return result;
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
}
}
return false;
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}
/**
* Test a video with an empty comment
*/
public static class EmptyComment {
private static YoutubeCommentsExtractor extractor;
private final static String url = "https://www.youtube.com/watch?v=VM_6n762j6M";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
assertTrue(Utils.isBlank(c.getCommentText()));
} else {
assertFalse(Utils.isBlank(c.getCommentText()));
}
}
}
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java
View file

@ -81,6 +81,15 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
assertEquals("n8X9_MgEdCg", linkHandler.fromUrl("vnd.youtube://n8X9_MgEdCg").getId());
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://m.youtube.com/watch?v=-cdveCh1kQk)").getId());
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://www.youtube.com/watch?v=-cdveCh1kQk-").getId());
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=-cdveCh1kQkwhatever").getId());
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=O0EDx9WAelc]").getId());
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("https://youtu.be/-cdveCh1kQk)hello").getId());
assertEquals("OGS7c0-CmRs", linkHandler.fromUrl("https://YouTu.be/OGS7c0-CmRswhatever)").getId());
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("HTTPS://youtu.be/-cdveCh1kQk)").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("https://www.youtube.com/shorts/IOS2fqxwYbAhi").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId());
}
@Test
@ -101,6 +110,7 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/shorts/IOS2fqxwYbA"));
}
@Test

84
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View file

@ -11,12 +11,16 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.FileUtils.resolveTestResource;
/**
* Test for {@link YoutubeSubscriptionExtractor}
@ -34,54 +38,48 @@ public class YoutubeSubscriptionExtractorTest {
@Test
public void testFromInputStream() throws Exception {
File testFile = new File("extractor/src/test/resources/youtube_export_test.xml");
if (!testFile.exists()) testFile = new File("src/test/resources/youtube_export_test.xml");
final List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(
new FileInputStream(resolveTestResource("youtube_takeout_import_test.json")));
assertEquals(7, subscriptionItems.size());
List<SubscriptionItem> subscriptionItems = subscriptionExtractor.fromInputStream(new FileInputStream(testFile));
assertTrue("List doesn't have exactly 8 items (had " + subscriptionItems.size() + ")", subscriptionItems.size() == 8);
for (SubscriptionItem item : subscriptionItems) {
for (final SubscriptionItem item : subscriptionItems) {
assertNotNull(item.getName());
assertNotNull(item.getUrl());
assertTrue(urlHandler.acceptUrl(item.getUrl()));
assertFalse(item.getServiceId() == -1);
assertEquals(ServiceList.YouTube.getServiceId(), item.getServiceId());
}
}
@Test
public void testEmptySourceException() throws Exception {
String emptySource = "<opml version=\"1.1\"><body>" +
"<outline text=\"Testing\" title=\"123\" />" +
"</body></opml>";
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(emptySource.getBytes("UTF-8")));
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
assertTrue(items.isEmpty());
}
@Test
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
String channelId = "AA0AaAa0AaaaAAAAAA0aa0AA";
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_id=" + channelId + "\" />" +
"</outline></body></opml>";
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue("List doesn't have exactly 1 item (had " + items.size() + ")", items.size() == 1);
assertTrue("Item does not have an empty title (had \"" + items.get(0).getName() + "\")", items.get(0).getName().isEmpty());
assertTrue("Item does not have the right channel id \"" + channelId + "\" (the whole url is \"" + items.get(0).getUrl() + "\")", items.get(0).getUrl().endsWith(channelId));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
assertEquals("", items.get(0).getName());
}
@Test
public void testSubscriptionWithInvalidUrlInSource() throws Exception {
String source = "<opml version=\"1.1\"><body><outline text=\"YouTube Subscriptions\" title=\"YouTube Subscriptions\">" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"https://www.youtube.com/feeds/videos.xml?channel_not_id=|||||||\"/>" +
"<outline text=\"fail\" title=\"fail\" type=\"rss\" xmlUgrl=\"invalidTag\"/>" +
"<outline text=\"invalid\" title=\"url\" type=\"rss\" xmlUrl=\"\"/>" +
"<outline text=\"\" title=\"\" type=\"rss\" xmlUrl=\"\"/>" +
"</outline></body></opml>";
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(new ByteArrayInputStream(source.getBytes("UTF-8")));
assertTrue(items.isEmpty());
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
assertEquals("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg", items.get(0).getUrl());
assertEquals("name2", items.get(0).getName());
}
@Test
@ -89,26 +87,26 @@ public class YoutubeSubscriptionExtractorTest {
List<String> invalidList = Arrays.asList(
"<xml><notvalid></notvalid></xml>",
"<opml><notvalid></notvalid></opml>",
"<opml><body></body></opml>",
"{\"a\":\"b\"}",
"[{}]",
"[\"\", 5]",
"[{\"snippet\":{\"title\":\"name\"}}]",
"[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"}}}]",
"",
null,
"\uD83D\uDC28\uD83D\uDC28\uD83D\uDC28",
"gibberish");
for (String invalidContent : invalidList) {
try {
if (invalidContent != null) {
byte[] bytes = invalidContent.getBytes("UTF-8");
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} else {
subscriptionExtractor.fromInputStream(null);
fail("Extracting from null String didn't throw an exception");
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {
boolean correctType = e instanceof SubscriptionExtractor.InvalidSourceException;
if (!correctType) {
e.printStackTrace();
}
} catch (Exception e) {
// System.out.println(" -> " + e);
boolean isExpectedException = e instanceof SubscriptionExtractor.InvalidSourceException;
assertTrue("\"" + e.getClass().getSimpleName() + "\" is not the expected exception", isExpectedException);
assertTrue(e.getClass().getSimpleName() + " is not InvalidSourceException", correctType);
}
}
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java
View file

@ -153,8 +153,8 @@ public class YoutubeMusicSearchExtractorTest {
public static class CorrectedSearch extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "duo lipa";
private static final String EXPECTED_SUGGESTION = "dua lipa";
private static final String QUERY = "nocopyrigh sounds";
private static final String EXPECTED_SUGGESTION = "nocopyrightsounds";
@BeforeClass
public static void setUp() throws Exception {

157
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View file

@ -1,144 +1,53 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Arrays;
import java.util.List;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamLinkHandlerFactory}
*/
public class YoutubeStreamExtractorAgeRestrictedTest {
public static final String HTTPS = "https://";
private static YoutubeStreamExtractor extractor;
public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
private static final String ID = "MmBeUZqv1QA";
private static final int TIMESTAMP = 196;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=MmBeUZqv1QA");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "FINGERING PORNSTARS @ AVN Expo 2017 In Las Vegas!"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertEquals(extractor.getTimeStamp() + "", "174");
extractor = YouTube.getStreamExtractor("https://youtube.com/embed/FmG385_uUys?start=174");
assertEquals(extractor.getTimeStamp() + "", "174");
}
@Test
public void testGetAgeLimit() throws ParsingException {
assertEquals(18, extractor.getAgeLimit());
}
@Test
public void testGetName() throws ParsingException {
assertNotNull("name is null", extractor.getName());
assertFalse("name is empty", extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(1790, extractor.getLength());
}
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getViewCount() > 0);
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
assertEquals("2017-01-25", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-01-25"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audio streams are not always necessary
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
List<VideoStream> streams = new ArrayList<>();
streams.addAll(extractor.getVideoStreams());
streams.addAll(extractor.getVideoOnlyStreams());
assertTrue(Integer.toString(streams.size()), streams.size() > 0);
for (VideoStream s : streams) {
assertTrue(s.getUrl(),
s.getUrl().contains(HTTPS));
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
}
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "EpicFiveTV"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCuPUHlLP5POZphOIrjrNxiw"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("http://instagram.com/Ruben_Sole", "AVN"); }
@Override public long expectedLength() { return 1790; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 28500000; }
@Nullable @Override public String expectedUploadDate() { return "2017-01-25 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2017-01-25"; }
@Override public long expectedLikeCountAtLeast() { return 149000; }
@Override public long expectedDislikeCountAtLeast() { return 38000; }
@Override public boolean expectedHasRelatedStreams() { return false; } // no related videos (!)
@Override public int expectedAgeLimit() { return 18; }
@Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
@Override public boolean expectedHasSubtitles() { return false; }
}

138
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View file

@ -1,134 +1,54 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Arrays;
import java.util.List;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamLinkHandlerFactory}
*/
public class YoutubeStreamExtractorControversialTest {
private static YoutubeStreamExtractor extractor;
public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
private static final String ID = "T4XJQO3qol8";
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Burning Everyone's Koran"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetValidTimeStamp() throws IOException, ExtractionException {
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertEquals(extractor.getTimeStamp() + "", "174");
}
@Test
@Ignore
public void testGetAgeLimit() throws ParsingException {
assertEquals(18, extractor.getAgeLimit());
}
@Test
public void testGetName() throws ParsingException {
assertNotNull("name is null", extractor.getName());
assertFalse("name is empty", extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(219, extractor.getLength());
}
@Test
public void testGetViews() throws ParsingException {
assertTrue(extractor.getViewCount() > 0);
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
assertEquals("2010-09-09", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2010-09-09"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws IOException, ExtractionException {
// audio streams are not always necessary
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws IOException, ExtractionException {
List<VideoStream> streams = new ArrayList<>();
streams.addAll(extractor.getVideoStreams());
streams.addAll(extractor.getVideoOnlyStreams());
assertTrue(streams.size() > 0);
}
@Test
public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
// Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=T4XJQO3qol8) set in the setUp() method has at least auto-generated (English) captions
assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Amazing Atheist"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html",
"freedom");
}
@Override public long expectedLength() { return 219; }
@Override public long expectedViewCountAtLeast() { return 285000; }
@Nullable @Override public String expectedUploadDate() { return "2010-09-09 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
@Override public long expectedLikeCountAtLeast() { return 13300; }
@Override public long expectedDislikeCountAtLeast() { return 2600; }
}

324
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View file

@ -1,35 +1,22 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Arrays;
import java.util.List;
import static java.util.Objects.requireNonNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import javax.annotation.Nullable;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/*
@ -51,11 +38,8 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Test for {@link StreamExtractor}
*/
public class YoutubeStreamExtractorDefaultTest {
static final String BASE_URL = "https://www.youtube.com/watch?v=";
public static class NotAvailable {
@BeforeClass
@ -66,266 +50,120 @@ public class YoutubeStreamExtractorDefaultTest {
@Test(expected = ContentNotAvailableException.class)
public void nonExistentFetch() throws Exception {
final StreamExtractor extractor =
YouTube.getStreamExtractor("https://www.youtube.com/watch?v=don-t-exist");
YouTube.getStreamExtractor(BASE_URL + "don-t-exist");
extractor.fetchPage();
}
@Test(expected = ParsingException.class)
public void invalidId() throws Exception {
final StreamExtractor extractor =
YouTube.getStreamExtractor("https://www.youtube.com/watch?v=INVALID_ID_INVALID_ID");
YouTube.getStreamExtractor(BASE_URL + "INVALID_ID_INVALID_ID");
extractor.fetchPage();
}
}
/**
* Test for {@link StreamExtractor}
*/
public static class AdeleHello {
private static YoutubeStreamExtractor extractor;
public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest {
private static final String ID = "7PIMiDcwNvc";
private static final int TIMESTAMP = 17;
private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetValidTimeStamp() throws ExtractionException {
StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
assertEquals(extractor.getTimeStamp() + "", "174");
}
@Test
public void testGetTitle() throws ParsingException {
assertFalse(extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("http://adele.com"));
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(367, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
Long count = extractor.getViewCount();
assertTrue(Long.toString(count), count >= /* specific to that video */ 1220025784);
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2015-10-22", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2015-10-22"));
assertEquals(instance, requireNonNull(extractor.getUploadDate()).date());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
String url = extractor.getUploaderUrl();
if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") &&
!url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) {
fail("Uploader url is neither the music channel one nor the Vevo one");
}
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
}
}
@Test
public void testStreamType() throws ParsingException {
assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM);
}
@Test
public void testGetDashMpd() throws ParsingException {
// we dont expect this particular video to have a DASH file. For this purpouse we use a different test class.
assertTrue(extractor.getDashMpdUrl(),
extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty());
}
@Test
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
@Test
public void testGetLikeCount() throws ParsingException {
long likeCount = extractor.getLikeCount();
assertTrue("" + likeCount, likeCount >= 15000000);
}
@Test
public void testGetDislikeCount() throws ParsingException {
long dislikeCount = extractor.getDislikeCount();
assertTrue("" + dislikeCount, dislikeCount >= 818000);
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "PewDiePie"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
"https://www.handcraftpictures.com/");
}
@Override public long expectedLength() { return 381; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 26682500; }
@Nullable @Override public String expectedUploadDate() { return "2019-08-24 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-08-24"; }
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
}
public static class DescriptionTestPewdiepie {
private static YoutubeStreamExtractor extractor;
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
private static final String ID = "cV5TjZCJkuA";
private static final String URL = BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "This Smartphone Changes Everything..."; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.reddit.com/r/PewdiepieSubmissions/"));
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng"));
assertTrue(extractor.getDescription().getContent().contains("https://usa.clutchchairz.com/product/pewdiepie-edition-throttle-series/"));
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Unbox Therapy"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
"https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
"https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
"https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34");
}
@Override public long expectedLength() { return 434; }
@Override public long expectedViewCountAtLeast() { return 21229200; }
@Nullable @Override public String expectedUploadDate() { return "2018-06-19 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
}
public static class DescriptionTestUnboxing {
private static YoutubeStreamExtractor extractor;
public static class RatingsDisabledTest extends DefaultStreamExtractorTest {
private static final String ID = "HRKu0cvrr_o";
private static final int TIMESTAMP = 17;
private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=cV5TjZCJkuA");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetFullLinksInDescription() throws ParsingException {
final String description = extractor.getDescription().getContent();
assertTrue(description.contains("https://www.youtube.com/watch?v=X7FLCHVXpsA&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=Lqv6G0pDNnw&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=XxaRBPyrnBU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
assertTrue(description.contains("https://www.youtube.com/watch?v=U-9tUEOFKNU&amp;list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"));
}
}
public static class RatingsDisabledTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o");
extractor.fetchPage();
}
@Test
public void testGetLikeCount() throws ParsingException {
assertEquals(-1, extractor.getLikeCount());
}
@Test
public void testGetDislikeCount() throws ParsingException {
assertEquals(-1, extractor.getDislikeCount());
}
}
public static class FramesTest {
private static YoutubeStreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=HoK9shIJ2xQ");
extractor.fetchPage();
}
@Test
public void testGetFrames() throws ExtractionException {
final List<Frameset> frames = extractor.getFrames();
assertNotNull(frames);
assertFalse(frames.isEmpty());
for (final Frameset f : frames) {
for (final String url : f.getUrls()) {
ExtractorAsserts.assertIsValidUrl(url);
ExtractorAsserts.assertIsSecureUrl(url);
}
}
}
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "YouTuber PrinceOfFALLEN"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCQT2yul0lr6Ie9qNQNmw-sg"; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("dislikes", "Alpha", "wrong"); }
@Override public long expectedLength() { return 84; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 190; }
@Nullable @Override public String expectedUploadDate() { return "2019-01-02 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
}
}

156
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View file

@ -1,139 +1,55 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorLivestreamTest {
private static YoutubeStreamExtractor extractor;
public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
private static final String ID = "5qap5aO4i9A";
private static final int TIMESTAMP = 1737;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=5qap5aO4i9A");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "lofi hip hop radio - beats to relax/study to"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetTitle() throws ParsingException {
assertFalse(extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://bit.ly/chilledcow-playlists"));
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(0, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
long count = extractor.getViewCount();
assertTrue(Long.toString(count), count > -1);
}
@Test
public void testGetUploadDate() throws ParsingException {
assertNull(extractor.getUploadDate());
assertNull(extractor.getTextualUploadDate());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws ExtractionException {
assertFalse(extractor.getAudioStreams().isEmpty());
}
@Test
public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() <= 0x100);
}
}
@Test
public void testStreamType() throws ParsingException {
assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM);
}
@Test
public void testGetDashMpd() throws ParsingException {
assertTrue(extractor.getDashMpdUrl().startsWith("https://manifest.googlevideo.com/api/manifest/dash/"));
}
@Test
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() {
assertTrue(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() {
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
@Override public StreamType expectedStreamType() { return StreamType.LIVE_STREAM; }
@Override public String expectedUploaderName() { return "ChilledCow"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://bit.ly/chilledcow-playlists",
"https://bit.ly/chilledcow-submissions");
}
@Override public long expectedLength() { return 0; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 0; }
@Nullable @Override public String expectedUploadDate() { return "2020-02-22 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2020-02-22"; }
@Override public long expectedLikeCountAtLeast() { return 825000; }
@Override public long expectedDislikeCountAtLeast() { return 15600; }
@Override public boolean expectedHasSubtitles() { return false; }
@Nullable @Override public String expectedDashMpdUrlContains() { return "https://manifest.googlevideo.com/api/manifest/dash/"; }
@Override public boolean expectedHasFrames() { return false; }
}

169
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View file

@ -1,161 +1,50 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorUnlistedTest {
private static YoutubeStreamExtractor extractor;
public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
static final String ID = "udsB8KnIJTg";
static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=udsB8KnIJTg");
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@Test
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(extractor.getTimeStamp() + "",
extractor.getTimeStamp() <= 0);
}
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Praise the Casual: Ein Neuling trifft Dark Souls - Folge 5"; }
@Override public String expectedId() { return ID; }
@Override public String expectedUrlContains() { return URL; }
@Override public String expectedOriginalUrlContains() { return URL; }
@Test
public void testGetTitle() throws ParsingException {
assertFalse(extractor.getName().isEmpty());
}
@Test
public void testGetDescription() throws ParsingException {
assertNotNull(extractor.getDescription());
assertFalse(extractor.getDescription().getContent().isEmpty());
}
@Test
public void testGetFullLinksInDescription() throws ParsingException {
assertTrue(extractor.getDescription().getContent().contains("https://www.youtube.com/user/Roccowschiptune"));
}
@Test
public void testGetUploaderName() throws ParsingException {
assertNotNull(extractor.getUploaderName());
assertFalse(extractor.getUploaderName().isEmpty());
}
@Test
public void testGetLength() throws ParsingException {
assertEquals(2488, extractor.getLength());
}
@Test
public void testGetViewCount() throws ParsingException {
long count = extractor.getViewCount();
assertTrue(Long.toString(count), count >= /* specific to that video */ 1225);
}
@Test
public void testGetTextualUploadDate() throws ParsingException {
Assert.assertEquals("2017-09-22", extractor.getTextualUploadDate());
}
@Test
public void testGetUploadDate() throws ParsingException, ParseException {
final Calendar instance = Calendar.getInstance();
instance.setTime(new SimpleDateFormat("yyyy-MM-dd").parse("2017-09-22"));
assertNotNull(extractor.getUploadDate());
assertEquals(instance, extractor.getUploadDate().date());
}
@Test
public void testGetUploaderUrl() throws ParsingException {
assertEquals("https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw", extractor.getUploaderUrl());
}
@Test
public void testGetThumbnailUrl() throws ParsingException {
assertIsSecureUrl(extractor.getThumbnailUrl());
}
@Test
public void testGetUploaderAvatarUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderAvatarUrl());
}
@Test
public void testGetAudioStreams() throws ExtractionException {
List<AudioStream> audioStreams = extractor.getAudioStreams();
assertFalse(audioStreams.isEmpty());
for (AudioStream s : audioStreams) {
assertIsSecureUrl(s.url);
assertTrue(Integer.toString(s.getFormatId()),
0x100 <= s.getFormatId() && s.getFormatId() < 0x1000);
}
}
@Test
public void testGetVideoStreams() throws ExtractionException {
for (VideoStream s : extractor.getVideoStreams()) {
assertIsSecureUrl(s.url);
assertTrue(s.resolution.length() > 0);
assertTrue(Integer.toString(s.getFormatId()),
0 <= s.getFormatId() && s.getFormatId() < 0x100);
}
}
@Test
public void testStreamType() throws ParsingException {
assertSame(StreamType.VIDEO_STREAM, extractor.getStreamType());
}
@Test
public void testGetRelatedVideos() throws ExtractionException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty());
}
@Test
public void testGetSubtitlesListDefault() {
assertFalse(extractor.getSubtitlesDefault().isEmpty());
}
@Test
public void testGetSubtitlesList() {
assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
}
@Test
public void testGetLikeCount() throws ParsingException {
long likeCount = extractor.getLikeCount();
assertTrue("" + likeCount, likeCount >= 96);
}
@Test
public void testGetDislikeCount() throws ParsingException {
long dislikeCount = extractor.getDislikeCount();
assertTrue("" + dislikeCount, dislikeCount >= 0);
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "Hooked"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCPysfiuOv4VKBeXFFPhKXyw"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("https://www.youtube.com/user/Roccowschiptune",
"https://www.facebook.com/HookedMagazinDE");
}
@Override public long expectedLength() { return 2488; }
@Override public long expectedViewCountAtLeast() { return 1500; }
@Nullable @Override public String expectedUploadDate() { return "2017-09-22 00:00:00.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2017-09-22"; }
@Override public long expectedLikeCountAtLeast() { return 110; }
@Override public long expectedDislikeCountAtLeast() { return 0; }
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java
View file

@ -21,4 +21,28 @@ public class UtilsTest {
public void testJoin() {
assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
}
@Test
public void testGetBaseUrl() throws ParsingException {
assertEquals("https://www.youtube.com", Utils.getBaseUrl("https://www.youtube.com/watch?v=Hu80uDzh8RY"));
assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube:jZViOEv90dI"));
assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg"));
assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
}
@Test
public void testFollowGoogleRedirect() {
assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY",
Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"));
assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA",
Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=video&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3D0b6cFWG45kA"));
assertEquals("https://soundcloud.com/ciaoproduction",
Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="));
assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz",
Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz"));
assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello",
Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello"));
}
}

23
extractor/src/test/resources/youtube_export_test.xml
View file

@ -1,23 +0,0 @@
<opml version="1.1">
<body>
<outline text="YouTube Subscriptions" title="YouTube Subscriptions">
<outline text="Kurzgesagt – In a Nutshell" title="Kurzgesagt – In a Nutshell" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="CaptainDisillusion" title="CaptainDisillusion" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCEOXxzW2vU0P-0THehuIIeg"/>
<outline text="TED" title="TED" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCAuUUnT6oDeKwE6v1NGQxug"/>
<outline text="Gorillaz" title="Gorillaz" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCfIXdjDQH9Fau7y99_Orpjw"/>
<outline text="ElectroBOOM" title="ElectroBOOM" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCJ0-OtVpF0wOKEqT2Z1HEtA"/>
<outline text="ⓤⓝⓘⓒⓞⓓⓔ" title="ⓤⓝⓘⓒⓞⓓⓔ" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="中文" title="中文" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
<outline text="हिंदी" title="हिंदी" type="rss"
xmlUrl="https://www.youtube.com/feeds/videos.xml?channel_id=UCsXVk37bltHxD1rDPwtNM8Q"/>
</outline>
</body>
</opml>

211
extractor/src/test/resources/youtube_takeout_import_test.json Normal file
View file

@ -0,0 +1,211 @@
[ {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "The official YouTube home of Gorillaz.",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Gorillaz"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 3502
},
"etag" : "wUgip-X0qBlnjj0frSTwP6B8XoY",
"id" : "qUAzBV8xkoLYOP-1gwzy63zpjj8SMTtDReGwIa2sHp8",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "The TED Talks channel features the best talks and performances from the TED Conference, where the world's leading thinkers and doers give the talk of their lives in 18 minutes (or less). Look for talks on Technology, Entertainment and Design -- plus science, business, global issues, the arts and more. You're welcome to link to or embed these videos, forward them to others and share these ideas with people you know.\n\nTED's videos may be used for non-commercial purposes under a Creative Commons License, Attribution–Non Commercial–No Derivatives (or the CC BY – NC – ND 4.0 International) and in accordance with our TED Talks Usage Policy (https://www.ted.com/about/our-organization/our-policies-terms/ted-talks-usage-policy). For more information on using TED for commercial purposes (e.g. employee learning, in a film or online course), please submit a Media Request at https://media-requests.ted.com",
"publishedAt" : "2020-11-01T17:24:11.769Z",
"resourceId" : {
"channelId" : "UCAuUUnT6oDeKwE6v1NGQxug",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-bonZt347bMc/AAAAAAAAAAI/AAAAAAAAAAA/lR8QEKnqqHk/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "TED"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 98
},
"etag" : "M3Hl6FQUAD3e-fH9pcvcE9aPSWQ",
"id" : "qUAzBV8xkoLYOP-1gwzy64Vo-PpWMPDyIYBM1JUfepk",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "In a world where the content of digital images and videos can no longer be taken at face value, an unlikely hero fights for the acceptance of truth.\r\n\r\nCaptain Disillusion guides children of all ages through the maze of visual fakery to the open spaces of reality and peace of mind.\r\n\r\nSubscribe to get fun and detailed explanations of current \"unbelievable\" viral videos that fool the masses!",
"publishedAt" : "2020-11-01T17:23:52.909Z",
"resourceId" : {
"channelId" : "UCEOXxzW2vU0P-0THehuIIeg",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-7f3hUYZP5Sc/AAAAAAAAAAI/AAAAAAAAAAA/4cpBHKDlbYQ/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Captain Disillusion"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 130
},
"etag" : "crkTVZbDHS3arRZErMaLMnNqtac",
"id" : "qUAzBV8xkoLYOP-1gwzy66EVopYHE34m06PVw8Pvheg",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "Videos explaining things with optimistic nihilism. \n\nWe are a small team who want to make science look beautiful. Because it is beautiful. \n\nCurrently we make one animation video per month. Follow us on Twitter, Facebook to get notified when a new one comes out.\n\nFAQ:\n \n- We do the videos with After Effects and Illustrator.",
"publishedAt" : "2020-11-01T17:23:39.659Z",
"resourceId" : {
"channelId" : "UCsXVk37bltHxD1rDPwtNM8Q",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UwENvFjc4vI/AAAAAAAAAAI/AAAAAAAAAAA/04dXvZ_jl0I/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "Kurzgesagt – In a Nutshell"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "ⓤⓝⓘⓒⓞⓓⓔ",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "ⓤⓝⓘⓒⓞⓓⓔ"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "中文",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "中文"
}
}, {
"contentDetails" : {
"activityType" : "all",
"newItemCount" : 0,
"totalItemCount" : 229
},
"etag" : "WRftgrOZw2rsNjNYhHG3s-VObgs",
"id" : "qUAzBV8xkoLYOP-1gwzy6yVWWYc7mBJxLNUEVlkNk8Y",
"kind" : "youtube#subscription",
"snippet" : {
"channelId" : "UC-lHJZR3Gqxm24_Vd_AJ5Yw",
"description" : "हिंदी",
"publishedAt" : "2020-11-01T17:24:34.498Z",
"resourceId" : {
"channelId" : "UCfIXdjDQH9Fau7y99_Orpjw",
"kind" : "youtube#channel"
},
"thumbnails" : {
"default" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium" : {
"url" : "https://yt3.ggpht.com/-UXcxdSDLo08/AAAAAAAAAAI/AAAAAAAAAAA/FP5NbxM7TzU/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
},
"title" : "हिंदी"
}
} ]

32
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/PatternsHolder.java
View file

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.timeago;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
@ -16,7 +17,7 @@ public abstract class PatternsHolder {
private final Collection<String> months;
private final Collection<String> years;
private final Map<TimeAgoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>();
private final Map<ChronoUnit, Map<String, Integer>> specialCases = new LinkedHashMap<>();
protected PatternsHolder(String wordSeparator, Collection<String> seconds, Collection<String> minutes,
Collection<String> hours, Collection<String> days,
@ -69,30 +70,25 @@ public abstract class PatternsHolder {
return years;
}
public Map<TimeAgoUnit, Map<String, Integer>> specialCases() {
public Map<ChronoUnit, Map<String, Integer>> specialCases() {
return specialCases;
}
protected void putSpecialCase(TimeAgoUnit unit, String caseText, int caseAmount) {
Map<String, Integer> item = specialCases.get(unit);
if (item == null) {
item = new LinkedHashMap<>();
specialCases.put(unit, item);
}
protected void putSpecialCase(ChronoUnit unit, String caseText, int caseAmount) {
Map<String, Integer> item = specialCases.computeIfAbsent(unit, k -> new LinkedHashMap<>());
item.put(caseText, caseAmount);
}
public Map<TimeAgoUnit, Collection<String>> asMap() {
final Map<TimeAgoUnit, Collection<String>> returnMap = new LinkedHashMap<>();
returnMap.put(TimeAgoUnit.SECONDS, seconds());
returnMap.put(TimeAgoUnit.MINUTES, minutes());
returnMap.put(TimeAgoUnit.HOURS, hours());
returnMap.put(TimeAgoUnit.DAYS, days());
returnMap.put(TimeAgoUnit.WEEKS, weeks());
returnMap.put(TimeAgoUnit.MONTHS, months());
returnMap.put(TimeAgoUnit.YEARS, years());
public Map<ChronoUnit, Collection<String>> asMap() {
final Map<ChronoUnit, Collection<String>> returnMap = new LinkedHashMap<>();
returnMap.put(ChronoUnit.SECONDS, seconds());
returnMap.put(ChronoUnit.MINUTES, minutes());
returnMap.put(ChronoUnit.HOURS, hours());
returnMap.put(ChronoUnit.DAYS, days());
returnMap.put(ChronoUnit.WEEKS, weeks());
returnMap.put(ChronoUnit.MONTHS, months());
returnMap.put(ChronoUnit.YEARS, years());
return returnMap;
}

11
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/TimeAgoUnit.java
View file

@ -1,11 +0,0 @@
package org.schabi.newpipe.extractor.timeago;
public enum TimeAgoUnit {
SECONDS,
MINUTES,
HOURS,
DAYS,
WEEKS,
MONTHS,
YEARS
}

13
timeago-parser/src/main/java/org/schabi/newpipe/extractor/timeago/patterns/iw.java
View file

@ -5,7 +5,8 @@
package org.schabi.newpipe.extractor.timeago.patterns;
import org.schabi.newpipe.extractor.timeago.PatternsHolder;
import org.schabi.newpipe.extractor.timeago.TimeAgoUnit;
import java.time.temporal.ChronoUnit;
public class iw extends PatternsHolder {
private static final String WORD_SEPARATOR = " ";
@ -26,10 +27,10 @@ public class iw extends PatternsHolder {
private iw() {
super(WORD_SEPARATOR, SECONDS, MINUTES, HOURS, DAYS, WEEKS, MONTHS, YEARS);
putSpecialCase(TimeAgoUnit.HOURS, "שעתיים", 2);
putSpecialCase(TimeAgoUnit.DAYS, "יומיים", 2);
putSpecialCase(TimeAgoUnit.WEEKS, "שבועיים", 2);
putSpecialCase(TimeAgoUnit.MONTHS, "חודשיים", 2);
putSpecialCase(TimeAgoUnit.YEARS, "שנתיים", 2);
putSpecialCase(ChronoUnit.HOURS, "שעתיים", 2);
putSpecialCase(ChronoUnit.DAYS, "יומיים", 2);
putSpecialCase(ChronoUnit.WEEKS, "שבועיים", 2);
putSpecialCase(ChronoUnit.MONTHS, "חודשיים", 2);
putSpecialCase(ChronoUnit.YEARS, "שנתיים", 2);
}
}