Merge pull request #291 from wb9688/yt-music-search
Add support for YouTube Music search
This commit is contained in:
Tobias Groza
GitHub
commit
b81e22ddc9
10 changed files with 881 additions and 55 deletions
|
|
@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor;
|
|||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
/**
|
||||
* Base class to extractors that have a list (e.g. playlists, users).
|
||||
*/
|
||||
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
|
||||
|
||||
/**
|
||||
* Constant that should be returned whenever
|
||||
* a list has an unknown number of items.
|
||||
*/
|
||||
public static final long ITEM_COUNT_UNKNOWN = -1;
|
||||
/**
|
||||
* Constant that should be returned whenever a list has an
|
||||
* infinite number of items. For example a YouTube mix.
|
||||
*/
|
||||
public static final long ITEM_COUNT_INFINITE = -2;
|
||||
/**
|
||||
* Constant that should be returned whenever a list
|
||||
* has an unknown number of items bigger than 100.
|
||||
*/
|
||||
public static final long ITEM_COUNT_MORE_THAN_100 = -3;
|
||||
|
||||
public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
|||
import org.schabi.newpipe.extractor.feed.FeedExtractor;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
|
||||
import org.schabi.newpipe.extractor.kiosk.KioskList;
|
||||
import org.schabi.newpipe.extractor.linkhandler.*;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.localization.ContentCountry;
|
||||
import org.schabi.newpipe.extractor.localization.Localization;
|
||||
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
|
||||
import org.schabi.newpipe.extractor.stream.StreamExtractor;
|
||||
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
|
||||
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
|
||||
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
|
||||
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
|
||||
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
|
||||
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
|
||||
|
||||
/*
|
||||
* Created by Christian Schabesberger on 23.08.15.
|
||||
|
|
@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService {
|
|||
|
||||
@Override
|
||||
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
|
||||
return new YoutubeSearchExtractor(this, query);
|
||||
final List<String> contentFilters = query.getContentFilters();
|
||||
|
||||
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) {
|
||||
return new YoutubeMusicSearchExtractor(this, query);
|
||||
} else {
|
||||
return new YoutubeSearchExtractor(this, query);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,494 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.extractors;
|
||||
|
||||
import com.grack.nanojson.JsonArray;
|
||||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
||||
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
|
||||
import org.schabi.newpipe.extractor.localization.DateWrapper;
|
||||
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
|
||||
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
|
||||
import org.schabi.newpipe.extractor.utils.Utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
|
||||
|
||||
public class YoutubeMusicSearchExtractor extends SearchExtractor {
|
||||
private JsonObject initialData;
|
||||
|
||||
public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
|
||||
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
|
||||
|
||||
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0];
|
||||
|
||||
final String params;
|
||||
|
||||
switch (getLinkHandler().getContentFilters().get(0)) {
|
||||
case MUSIC_SONGS:
|
||||
params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D";
|
||||
break;
|
||||
case MUSIC_VIDEOS:
|
||||
params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D";
|
||||
break;
|
||||
case MUSIC_ALBUMS:
|
||||
params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D";
|
||||
break;
|
||||
case MUSIC_PLAYLISTS:
|
||||
params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D";
|
||||
break;
|
||||
case MUSIC_ARTISTS:
|
||||
params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D";
|
||||
break;
|
||||
default:
|
||||
params = null;
|
||||
break;
|
||||
}
|
||||
|
||||
// @formatter:off
|
||||
byte[] json = JsonWriter.string()
|
||||
.object()
|
||||
.object("context")
|
||||
.object("client")
|
||||
.value("clientName", "WEB_REMIX")
|
||||
.value("clientVersion", youtubeMusicKeys[2])
|
||||
.value("hl", "en")
|
||||
.value("gl", getExtractorContentCountry().getCountryCode())
|
||||
.array("experimentIds").end()
|
||||
.value("experimentsToken", "")
|
||||
.value("utcOffsetMinutes", 0)
|
||||
.object("locationInfo").end()
|
||||
.object("musicAppInfo").end()
|
||||
.end()
|
||||
.object("capabilities").end()
|
||||
.object("request")
|
||||
.array("internalExperimentFlags").end()
|
||||
.object("sessionIndex").end()
|
||||
.end()
|
||||
.object("activePlayers").end()
|
||||
.object("user")
|
||||
.value("enableSafetyMode", false)
|
||||
.end()
|
||||
.end()
|
||||
.value("query", getSearchString())
|
||||
.value("params", params)
|
||||
.end().done().getBytes("UTF-8");
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
|
||||
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
|
||||
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
|
||||
headers.put("Referer", Collections.singletonList("music.youtube.com"));
|
||||
headers.put("Content-Type", Collections.singletonList("application/json"));
|
||||
|
||||
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json));
|
||||
|
||||
try {
|
||||
initialData = JsonParser.object().from(responseBody);
|
||||
} catch (JsonParserException e) {
|
||||
throw new ParsingException("Could not parse JSON", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
return super.getUrl();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSearchSuggestion() throws ParsingException {
|
||||
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
|
||||
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
|
||||
if (itemSectionRenderer == null) {
|
||||
return "";
|
||||
}
|
||||
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
|
||||
.getObject(0).getObject("didYouMeanRenderer");
|
||||
if (didYouMeanRenderer == null) {
|
||||
return "";
|
||||
}
|
||||
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException, IOException {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
|
||||
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
|
||||
|
||||
for (Object content : contents) {
|
||||
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
|
||||
collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents"));
|
||||
}
|
||||
}
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageUrl());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNextPageUrl() throws ExtractionException, IOException {
|
||||
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
|
||||
|
||||
for (Object content : contents) {
|
||||
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
|
||||
return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations"));
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
|
||||
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
|
||||
|
||||
// @formatter:off
|
||||
byte[] json = JsonWriter.string()
|
||||
.object()
|
||||
.object("context")
|
||||
.object("client")
|
||||
.value("clientName", "WEB_REMIX")
|
||||
.value("clientVersion", youtubeMusicKeys[2])
|
||||
.value("hl", "en")
|
||||
.value("gl", getExtractorContentCountry().getCountryCode())
|
||||
.array("experimentIds").end()
|
||||
.value("experimentsToken", "")
|
||||
.value("utcOffsetMinutes", 0)
|
||||
.object("locationInfo").end()
|
||||
.object("musicAppInfo").end()
|
||||
.end()
|
||||
.object("capabilities").end()
|
||||
.object("request")
|
||||
.array("internalExperimentFlags").end()
|
||||
.object("sessionIndex").end()
|
||||
.end()
|
||||
.object("activePlayers").end()
|
||||
.object("user")
|
||||
.value("enableSafetyMode", false)
|
||||
.end()
|
||||
.end()
|
||||
.end().done().getBytes("UTF-8");
|
||||
// @formatter:on
|
||||
|
||||
final Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
|
||||
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
|
||||
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
|
||||
headers.put("Referer", Collections.singletonList("music.youtube.com"));
|
||||
headers.put("Content-Type", Collections.singletonList("application/json"));
|
||||
|
||||
final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json));
|
||||
|
||||
final JsonObject ajaxJson;
|
||||
try {
|
||||
ajaxJson = JsonParser.object().from(responseBody);
|
||||
} catch (JsonParserException e) {
|
||||
throw new ParsingException("Could not parse JSON", e);
|
||||
}
|
||||
|
||||
if (ajaxJson.getObject("continuationContents") == null) {
|
||||
return InfoItemsPage.emptyPage();
|
||||
}
|
||||
|
||||
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation");
|
||||
|
||||
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
|
||||
final JsonArray continuations = musicShelfContinuation.getArray("continuations");
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
|
||||
}
|
||||
|
||||
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
for (Object item : videos) {
|
||||
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer");
|
||||
if (info != null) {
|
||||
final String searchType = getLinkHandler().getContentFilters().get(0);
|
||||
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
|
||||
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
|
||||
if (url != null && !url.isEmpty()) {
|
||||
return url;
|
||||
}
|
||||
throw new ParsingException("Could not get url");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (name != null && !name.isEmpty()) {
|
||||
return name;
|
||||
}
|
||||
throw new ParsingException("Could not get name");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getDuration() throws ParsingException {
|
||||
final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (duration != null && !duration.isEmpty()) {
|
||||
return YoutubeParsingHelper.parseDurationString(duration);
|
||||
}
|
||||
throw new ParsingException("Could not get duration");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (name != null && !name.isEmpty()) {
|
||||
return name;
|
||||
}
|
||||
throw new ParsingException("Could not get uploader name");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderUrl() throws ParsingException {
|
||||
if (searchType.equals(MUSIC_VIDEOS)) {
|
||||
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items");
|
||||
for (Object item : items) {
|
||||
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer");
|
||||
if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) {
|
||||
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
} else {
|
||||
final JsonObject navigationEndpoint = info.getArray("flexColumns")
|
||||
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
|
||||
.getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint");
|
||||
|
||||
if (navigationEndpoint == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
|
||||
|
||||
if (url != null && !url.isEmpty()) {
|
||||
return url;
|
||||
}
|
||||
throw new ParsingException("Could not get uploader url");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTextualUploadDate() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DateWrapper getUploadDate() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getViewCount() throws ParsingException {
|
||||
if (searchType.equals(MUSIC_SONGS)) {
|
||||
return -1;
|
||||
}
|
||||
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (viewCount != null && !viewCount.isEmpty()) {
|
||||
return Utils.mixedNumberWordToLong(viewCount);
|
||||
}
|
||||
throw new ParsingException("Could not get view count");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
|
||||
.getObject("thumbnail").getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
} else if (searchType.equals(MUSIC_ARTISTS)) {
|
||||
collector.commit(new YoutubeChannelInfoItemExtractor(info) {
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
|
||||
.getObject("thumbnail").getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (name != null && !name.isEmpty()) {
|
||||
return name;
|
||||
}
|
||||
throw new ParsingException("Could not get name");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint"));
|
||||
if (url != null && !url.isEmpty()) {
|
||||
return url;
|
||||
}
|
||||
throw new ParsingException("Could not get url");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSubscriberCount() throws ParsingException {
|
||||
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (viewCount != null && !viewCount.isEmpty()) {
|
||||
return Utils.mixedNumberWordToLong(viewCount);
|
||||
}
|
||||
throw new ParsingException("Could not get subscriber count");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamCount() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
} else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) {
|
||||
collector.commit(new YoutubePlaylistInfoItemExtractor(info) {
|
||||
@Override
|
||||
public String getThumbnailUrl() throws ParsingException {
|
||||
try {
|
||||
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
|
||||
.getObject("thumbnail").getArray("thumbnails");
|
||||
// the last thumbnail is the one with the highest resolution
|
||||
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
|
||||
|
||||
return fixThumbnailUrl(url);
|
||||
} catch (Exception e) {
|
||||
throw new ParsingException("Could not get thumbnail url", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() throws ParsingException {
|
||||
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (name != null && !name.isEmpty()) {
|
||||
return name;
|
||||
}
|
||||
throw new ParsingException("Could not get name");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUrl() throws ParsingException {
|
||||
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
|
||||
if (url != null && !url.isEmpty()) {
|
||||
return url;
|
||||
}
|
||||
throw new ParsingException("Could not get url");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUploaderName() throws ParsingException {
|
||||
final String name;
|
||||
if (searchType.equals(MUSIC_ALBUMS)) {
|
||||
name = getTextFromObject(info.getArray("flexColumns").getObject(2)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
} else {
|
||||
name = getTextFromObject(info.getArray("flexColumns").getObject(1)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
}
|
||||
if (name != null && !name.isEmpty()) {
|
||||
return name;
|
||||
}
|
||||
throw new ParsingException("Could not get uploader name");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getStreamCount() throws ParsingException {
|
||||
if (searchType.equals(MUSIC_ALBUMS)) {
|
||||
return ITEM_COUNT_UNKNOWN;
|
||||
}
|
||||
final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
|
||||
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
|
||||
if (count != null && !count.isEmpty()) {
|
||||
if (count.contains("100+")) {
|
||||
return ITEM_COUNT_MORE_THAN_100;
|
||||
} else {
|
||||
return Long.parseLong(Utils.removeNonDigitCharacters(count));
|
||||
}
|
||||
}
|
||||
throw new ParsingException("Could not get count");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
|
||||
if (continuations == null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
|
||||
final String continuation = nextContinuationData.getString("continuation");
|
||||
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
|
||||
|
||||
return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation
|
||||
+ "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0];
|
||||
}
|
||||
}
|
||||
|
|
@ -43,12 +43,12 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP
|
|||
public class YoutubeSearchExtractor extends SearchExtractor {
|
||||
private JsonObject initialData;
|
||||
|
||||
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
|
||||
public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
|
||||
super(service, linkHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
|
||||
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
|
||||
final String url = getUrl() + "&pbj=1";
|
||||
|
||||
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
|
||||
|
|
@ -64,23 +64,23 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
|
||||
@Override
|
||||
public String getSearchSuggestion() throws ParsingException {
|
||||
JsonObject showingResultsForRenderer = initialData.getObject("contents")
|
||||
final JsonObject showingResultsForRenderer = initialData.getObject("contents")
|
||||
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
|
||||
.getObject("sectionListRenderer").getArray("contents").getObject(0)
|
||||
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
|
||||
.getObject("showingResultsForRenderer");
|
||||
if (showingResultsForRenderer == null) {
|
||||
return "";
|
||||
} else {
|
||||
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
|
||||
}
|
||||
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
|
||||
|
||||
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
|
||||
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
|
||||
|
||||
for (Object section : sections) {
|
||||
|
|
@ -98,7 +98,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
|
||||
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
|
||||
if (pageUrl == null || pageUrl.isEmpty()) {
|
||||
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
|
||||
}
|
||||
|
|
@ -106,17 +106,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
|
||||
|
||||
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
|
||||
final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
|
||||
.getObject("continuationContents").getObject("itemSectionContinuation");
|
||||
|
||||
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
|
||||
final JsonArray continuations = itemSectionRenderer.getArray("continuations");
|
||||
|
||||
return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")));
|
||||
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
|
||||
}
|
||||
|
||||
private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException {
|
||||
collector.reset();
|
||||
|
||||
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
|
||||
final TimeAgoParser timeAgoParser = getTimeAgoParser();
|
||||
|
||||
for (Object item : videos) {
|
||||
|
|
@ -133,14 +132,15 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
}
|
||||
}
|
||||
|
||||
private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException {
|
||||
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException {
|
||||
if (continuations == null) {
|
||||
return "";
|
||||
}
|
||||
|
||||
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
|
||||
String continuation = nextContinuationData.getString("continuation");
|
||||
String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
|
||||
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
|
||||
final String continuation = nextContinuationData.getString("continuation");
|
||||
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
|
||||
|
||||
return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
|
||||
+ "&itct=" + clickTrackingParams;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import com.grack.nanojson.JsonArray;
|
|||
import com.grack.nanojson.JsonObject;
|
||||
import com.grack.nanojson.JsonParser;
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import com.grack.nanojson.JsonWriter;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
|
|
@ -18,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.text.ParseException;
|
||||
|
|
@ -62,6 +65,9 @@ public class YoutubeParsingHelper {
|
|||
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
|
||||
private static String clientVersion;
|
||||
|
||||
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
|
||||
private static String[] youtubeMusicKeys;
|
||||
|
||||
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
|
||||
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
|
||||
|
||||
|
|
@ -196,11 +202,7 @@ public class YoutubeParsingHelper {
|
|||
*/
|
||||
public static String getClientVersion() throws IOException, ExtractionException {
|
||||
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
|
||||
|
||||
if (isHardcodedClientVersionValid()) {
|
||||
clientVersion = HARDCODED_CLIENT_VERSION;
|
||||
return clientVersion;
|
||||
}
|
||||
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
|
||||
|
||||
final String url = "https://www.youtube.com/results?search_query=test";
|
||||
final String html = getDownloader().get(url).responseBody();
|
||||
|
|
@ -217,8 +219,7 @@ public class YoutubeParsingHelper {
|
|||
JsonObject p = (JsonObject) param;
|
||||
String key = p.getString("key");
|
||||
if (key != null && key.equals("cver")) {
|
||||
clientVersion = p.getString("value");
|
||||
return clientVersion;
|
||||
return clientVersion = p.getString("value");
|
||||
}
|
||||
}
|
||||
} else if (s.getString("service").equals("ECATCHER")) {
|
||||
|
|
@ -244,21 +245,94 @@ public class YoutubeParsingHelper {
|
|||
try {
|
||||
contextClientVersion = Parser.matchGroup1(pattern, html);
|
||||
if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
|
||||
clientVersion = contextClientVersion;
|
||||
return clientVersion;
|
||||
return clientVersion = contextClientVersion;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
if (shortClientVersion != null) {
|
||||
clientVersion = shortClientVersion;
|
||||
return clientVersion;
|
||||
return clientVersion = shortClientVersion;
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get client version");
|
||||
}
|
||||
|
||||
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
|
||||
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
|
||||
|
||||
// @formatter:off
|
||||
byte[] json = JsonWriter.string()
|
||||
.object()
|
||||
.object("context")
|
||||
.object("client")
|
||||
.value("clientName", "WEB_REMIX")
|
||||
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2])
|
||||
.value("hl", "en")
|
||||
.value("gl", "GB")
|
||||
.array("experimentIds").end()
|
||||
.value("experimentsToken", "")
|
||||
.value("utcOffsetMinutes", 0)
|
||||
.object("locationInfo").end()
|
||||
.object("musicAppInfo").end()
|
||||
.end()
|
||||
.object("capabilities").end()
|
||||
.object("request")
|
||||
.array("internalExperimentFlags").end()
|
||||
.object("sessionIndex").end()
|
||||
.end()
|
||||
.object("activePlayers").end()
|
||||
.object("user")
|
||||
.value("enableSafetyMode", false)
|
||||
.end()
|
||||
.end()
|
||||
.value("query", "test")
|
||||
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
|
||||
.end().done().getBytes("UTF-8");
|
||||
// @formatter:on
|
||||
|
||||
Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1]));
|
||||
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2]));
|
||||
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
|
||||
headers.put("Referer", Collections.singletonList("music.youtube.com"));
|
||||
headers.put("Content-Type", Collections.singletonList("application/json"));
|
||||
|
||||
String response = getDownloader().post(url, headers, json).responseBody();
|
||||
|
||||
return response.length() > 50; // ensure to have a valid response
|
||||
}
|
||||
|
||||
public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException {
|
||||
if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys;
|
||||
if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS;
|
||||
|
||||
final String url = "https://music.youtube.com/";
|
||||
final String html = getDownloader().get(url).responseBody();
|
||||
|
||||
String key;
|
||||
try {
|
||||
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||
} catch (Parser.RegexException e) {
|
||||
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||
}
|
||||
|
||||
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
|
||||
|
||||
String clientVersion;
|
||||
try {
|
||||
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
|
||||
} catch (Parser.RegexException e) {
|
||||
try {
|
||||
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
|
||||
} catch (Parser.RegexException ee) {
|
||||
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
|
||||
}
|
||||
}
|
||||
|
||||
return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
|
||||
}
|
||||
|
||||
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
|
||||
if (navigationEndpoint.getObject("urlEndpoint") != null) {
|
||||
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
|
||||
|
|
@ -303,6 +377,9 @@ public class YoutubeParsingHelper {
|
|||
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
|
||||
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
|
||||
return url.toString();
|
||||
} else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) {
|
||||
return "https://www.youtube.com/playlist?list=" +
|
||||
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -351,12 +428,8 @@ public class YoutubeParsingHelper {
|
|||
return thumbnailUrl;
|
||||
}
|
||||
|
||||
public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException {
|
||||
Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
|
||||
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
|
||||
final Response response = getDownloader().get(url, headers, localization);
|
||||
|
||||
public static String getValidJsonResponseBody(final Response response)
|
||||
throws ParsingException, MalformedURLException {
|
||||
if (response.responseCode() == 404) {
|
||||
throw new ContentNotAvailableException("Not found" +
|
||||
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
|
||||
|
|
@ -377,11 +450,24 @@ public class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
final String responseContentType = response.getHeader("Content-Type");
|
||||
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
|
||||
if (responseContentType != null
|
||||
&& responseContentType.toLowerCase().contains("text/html")) {
|
||||
throw new ParsingException("Got HTML document, expected JSON response" +
|
||||
" (latest url was: \"" + response.latestUrl() + "\")");
|
||||
}
|
||||
|
||||
return responseBody;
|
||||
}
|
||||
|
||||
public static JsonArray getJsonResponse(final String url, final Localization localization)
|
||||
throws IOException, ExtractionException {
|
||||
Map<String, List<String>> headers = new HashMap<>();
|
||||
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
|
||||
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
|
||||
final Response response = getDownloader().get(url, headers, localization);
|
||||
|
||||
final String responseBody = getValidJsonResponseBody(response);
|
||||
|
||||
try {
|
||||
return JsonParser.array().from(responseBody);
|
||||
} catch (JsonParserException e) {
|
||||
|
|
@ -393,6 +479,7 @@ public class YoutubeParsingHelper {
|
|||
* Shared alert detection function, multiple endpoints return the error similarly structured.
|
||||
* <p>
|
||||
* Will check if the object has an alert of the type "ERROR".
|
||||
* </p>
|
||||
*
|
||||
* @param initialData the object which will be checked if an alert is present
|
||||
* @throws ContentNotAvailableException if an alert is detected
|
||||
|
|
|
|||
|
|
@ -8,13 +8,21 @@ import java.net.URLEncoder;
|
|||
import java.util.List;
|
||||
|
||||
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
|
||||
|
||||
public static final String CHARSET_UTF_8 = "UTF-8";
|
||||
|
||||
public static final String ALL = "all";
|
||||
public static final String VIDEOS = "videos";
|
||||
public static final String CHANNELS = "channels";
|
||||
public static final String PLAYLISTS = "playlists";
|
||||
public static final String ALL = "all";
|
||||
|
||||
public static final String MUSIC_SONGS = "music_songs";
|
||||
public static final String MUSIC_VIDEOS = "music_videos";
|
||||
public static final String MUSIC_ALBUMS = "music_albums";
|
||||
public static final String MUSIC_PLAYLISTS = "music_playlists";
|
||||
public static final String MUSIC_ARTISTS = "music_artists";
|
||||
|
||||
private static final String SEARCH_URL = "https://www.youtube.com/results?search_query=";
|
||||
private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q=";
|
||||
|
||||
public static YoutubeSearchQueryHandlerFactory getInstance() {
|
||||
return new YoutubeSearchQueryHandlerFactory();
|
||||
|
|
@ -23,20 +31,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
|
|||
@Override
|
||||
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
|
||||
try {
|
||||
final String url = "https://www.youtube.com/results"
|
||||
+ "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
|
||||
if (contentFilters.size() > 0) {
|
||||
switch (contentFilters.get(0)) {
|
||||
case VIDEOS: return url + "&sp=EgIQAQ%253D%253D";
|
||||
case CHANNELS: return url + "&sp=EgIQAg%253D%253D";
|
||||
case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D";
|
||||
case ALL:
|
||||
default:
|
||||
break;
|
||||
case VIDEOS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
|
||||
case CHANNELS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
|
||||
case PLAYLISTS:
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
|
||||
case MUSIC_SONGS:
|
||||
case MUSIC_VIDEOS:
|
||||
case MUSIC_ALBUMS:
|
||||
case MUSIC_PLAYLISTS:
|
||||
case MUSIC_ARTISTS:
|
||||
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
}
|
||||
}
|
||||
|
||||
return url;
|
||||
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
throw new ParsingException("Could not encode query", e);
|
||||
}
|
||||
|
|
@ -48,6 +63,12 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
|
|||
ALL,
|
||||
VIDEOS,
|
||||
CHANNELS,
|
||||
PLAYLISTS};
|
||||
PLAYLISTS,
|
||||
MUSIC_SONGS,
|
||||
MUSIC_VIDEOS,
|
||||
MUSIC_ALBUMS,
|
||||
MUSIC_PLAYLISTS
|
||||
// MUSIC_ARTISTS
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,11 +37,14 @@ public final class DefaultTests {
|
|||
if (item instanceof StreamInfoItem) {
|
||||
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
|
||||
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
|
||||
assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
|
||||
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
|
||||
|
||||
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
|
||||
if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) {
|
||||
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
|
||||
assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
|
||||
}
|
||||
|
||||
assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM);
|
||||
assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
|
||||
|
||||
final String textualUploadDate = streamInfoItem.getTextualUploadDate();
|
||||
if (textualUploadDate != null && !textualUploadDate.isEmpty()) {
|
||||
|
|
|
|||
|
|
@ -22,4 +22,10 @@ public class YoutubeParsingHelperTest {
|
|||
assertTrue("Hardcoded client version is not valid anymore",
|
||||
YoutubeParsingHelper.isHardcodedClientVersionValid());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException {
|
||||
assertTrue("Hardcoded YouTube Music keys are not valid anymore",
|
||||
YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,153 @@
|
|||
package org.schabi.newpipe.extractor.services.youtube.search;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.schabi.newpipe.DownloaderTestImpl;
|
||||
import org.schabi.newpipe.extractor.InfoItem;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.extractor.StreamingService;
|
||||
import org.schabi.newpipe.extractor.search.SearchExtractor;
|
||||
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
|
||||
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
|
||||
|
||||
public class YoutubeMusicSearchExtractorTest {
|
||||
public static class MusicSongs extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "mocromaniac";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
|
||||
}
|
||||
|
||||
public static class MusicVideos extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "fresku";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
|
||||
}
|
||||
|
||||
public static class MusicAlbums extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "johnny sellah";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
|
||||
}
|
||||
|
||||
public static class MusicPlaylists extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "louivos";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
|
||||
}
|
||||
|
||||
@Ignore
|
||||
public static class MusicArtists extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "kevin";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return null; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
|
||||
}
|
||||
|
||||
public static class Suggestion extends DefaultSearchExtractorTest {
|
||||
private static SearchExtractor extractor;
|
||||
private static final String QUERY = "megaman x3";
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
NewPipe.init(DownloaderTestImpl.getInstance());
|
||||
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
|
||||
extractor.fetchPage();
|
||||
}
|
||||
|
||||
@Override public SearchExtractor extractor() { return extractor; }
|
||||
@Override public StreamingService expectedService() { return YouTube; }
|
||||
@Override public String expectedName() { return QUERY; }
|
||||
@Override public String expectedId() { return QUERY; }
|
||||
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
|
||||
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
|
||||
@Override public String expectedSearchString() { return QUERY; }
|
||||
@Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; }
|
||||
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
|
||||
}
|
||||
}
|
||||
|
|
@ -16,6 +16,12 @@ public class YoutubeSearchQHTest {
|
|||
assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
|
||||
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
|
||||
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
|
||||
|
||||
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -24,6 +30,9 @@ public class YoutubeSearchQHTest {
|
|||
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
|
||||
assertEquals(CHANNELS, YouTube.getSearchQHFactory()
|
||||
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
|
||||
|
||||
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
|
||||
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -36,16 +45,23 @@ public class YoutubeSearchQHTest {
|
|||
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
|
||||
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
|
||||
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
|
||||
|
||||
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
|
||||
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAvailableContentFilter() {
|
||||
final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter();
|
||||
assertEquals(4, contentFilter.length);
|
||||
assertEquals(8, contentFilter.length);
|
||||
assertEquals("all", contentFilter[0]);
|
||||
assertEquals("videos", contentFilter[1]);
|
||||
assertEquals("channels", contentFilter[2]);
|
||||
assertEquals("playlists", contentFilter[3]);
|
||||
assertEquals("music_songs", contentFilter[4]);
|
||||
assertEquals("music_videos", contentFilter[5]);
|
||||
assertEquals("music_albums", contentFilter[6]);
|
||||
assertEquals("music_playlists", contentFilter[7]);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
Loading…
Reference in a new issue