CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #996 from TeamNewPipe/feat/peeertube-playlists

Browse source 
[PeerTube] Support searching for playlists and channels
This commit is contained in:
Stypox 2023-01-02 15:04:09 +01:00 committed by GitHub
commit ce15f7cc50
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 175 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View file

@ -2,10 +2,14 @@ package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistInfoItemExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
@ -101,10 +105,16 @@ public final class PeertubeParsingHelper {
if (item.has("video")) {
item = item.getObject("video");
}
final boolean isPlaylistInfoItem = item.has("videosLength");
final boolean isChannelInfoItem = item.has("followersCount");
final PeertubeStreamInfoItemExtractor extractor;
final InfoItemExtractor extractor;
if (sepia) {
extractor = new PeertubeSepiaStreamInfoItemExtractor(item, baseUrl);
} else if (isPlaylistInfoItem) {
extractor = new PeertubePlaylistInfoItemExtractor(item, baseUrl);
} else if (isChannelInfoItem) {
extractor = new PeertubeChannelInfoItemExtractor(item, baseUrl);
} else {
extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
}

62
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelInfoItemExtractor.java Normal file
View file

@ -0,0 +1,62 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import java.util.Comparator;
public class PeertubeChannelInfoItemExtractor implements ChannelInfoItemExtractor {
final JsonObject item;
final JsonObject uploader;
final String baseUrl;
public PeertubeChannelInfoItemExtractor(@Nonnull final JsonObject item,
@Nonnull final String baseUrl) {
this.item = item;
this.uploader = item.getObject("uploader");
this.baseUrl = baseUrl;
}
@Override
public String getName() throws ParsingException {
return item.getString("displayName");
}
@Override
public String getUrl() throws ParsingException {
return item.getString("url");
}
@Override
public String getThumbnailUrl() throws ParsingException {
return item.getArray("avatars").stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.max(Comparator.comparingInt(avatar -> avatar.getInt("width")))
.map(avatar -> baseUrl + avatar.getString("path"))
.orElse(null);
}
@Override
public String getDescription() throws ParsingException {
return item.getString("description");
}
@Override
public long getSubscriberCount() throws ParsingException {
return item.getInt("followersCount");
}
@Override
public long getStreamCount() throws ParsingException {
return ChannelExtractor.ITEM_COUNT_UNKNOWN;
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
}

57
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistInfoItemExtractor.java Normal file
View file

@ -0,0 +1,57 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import javax.annotation.Nonnull;
public class PeertubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
final JsonObject item;
final JsonObject uploader;
final String baseUrl;
public PeertubePlaylistInfoItemExtractor(@Nonnull final JsonObject item,
@Nonnull final String baseUrl) {
this.item = item;
this.uploader = item.getObject("uploader");
this.baseUrl = baseUrl;
}
@Override
public String getName() throws ParsingException {
return item.getString("displayName");
}
@Override
public String getUrl() throws ParsingException {
return item.getString("url");
}
@Override
public String getThumbnailUrl() throws ParsingException {
return baseUrl + item.getString("thumbnailPath");
}
@Override
public String getUploaderName() throws ParsingException {
return uploader.getString("displayName");
}
@Override
public String getUploaderUrl() throws ParsingException {
return uploader.getString("url");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public long getStreamCount() throws ParsingException {
return item.getInt("videosLength");
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View file

@ -329,7 +329,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
throws UnsupportedEncodingException {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT_VIDEOS;
final StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
for (final String tag : tags) {

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubePlaylistLinkHandlerFactory.java
View file

@ -13,6 +13,7 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
private static final PeertubePlaylistLinkHandlerFactory INSTANCE
= new PeertubePlaylistLinkHandlerFactory();
private static final String ID_PATTERN = "(/videos/watch/playlist/|/w/p/)([^/?&#]*)";
private static final String API_ID_PATTERN = "/video-playlists/([^/?&#]*)";
private PeertubePlaylistLinkHandlerFactory() {
}
@ -38,7 +39,12 @@ public final class PeertubePlaylistLinkHandlerFactory extends ListLinkHandlerFac
@Override
public String getId(final String url) throws ParsingException {
return Parser.matchGroup(ID_PATTERN, url, 2);
try {
return Parser.matchGroup(ID_PATTERN, url, 2);
} catch (final ParsingException ignored) {
// might also be an API url, no reason to throw an exception here
}
return Parser.matchGroup1(API_ID_PATTERN, url);
}
@Override

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View file

@ -12,8 +12,12 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String PLAYLISTS = "playlists";
public static final String CHANNELS = "channels";
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
public static final String SEARCH_ENDPOINT = "/api/v1/search/videos";
public static final String SEARCH_ENDPOINT_PLAYLISTS = "/api/v1/search/video-playlists";
public static final String SEARCH_ENDPOINT_VIDEOS = "/api/v1/search/videos";
public static final String SEARCH_ENDPOINT_CHANNELS = "/api/v1/search/video-channels";
private PeertubeSearchQueryHandlerFactory() {
}
@ -41,7 +45,17 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
final String sortFilter,
final String baseUrl) throws ParsingException {
try {
return baseUrl + SEARCH_ENDPOINT + "?search=" + Utils.encodeUrlUtf8(searchString);
final String endpoint;
if (contentFilters.isEmpty()
|| contentFilters.get(0).equals(VIDEOS)
|| contentFilters.get(0).equals(SEPIA_VIDEOS)) {
endpoint = SEARCH_ENDPOINT_VIDEOS;
} else if (contentFilters.get(0).equals(CHANNELS)) {
endpoint = SEARCH_ENDPOINT_CHANNELS;
} else {
endpoint = SEARCH_ENDPOINT_PLAYLISTS;
}
return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}
@ -51,7 +65,9 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
public String[] getAvailableContentFilter() {
return new String[]{
VIDEOS,
SEPIA_VIDEOS
PLAYLISTS,
CHANNELS,
SEPIA_VIDEOS,
};
}
}

19
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java
View file

@ -18,13 +18,30 @@ public class PeertubeSearchQHTest {
}
@Test
public void testRegularValues() throws Exception {
void testVideoSearch() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
}
@Test
void testSepiaVideoSearch() throws Exception {
assertEquals("https://sepiasearch.org/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "").getUrl());
assertEquals("https://anotherpeertubeindex.com/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "", "https://anotherpeertubeindex.com").getUrl());
}
@Test
void testPlaylistSearch() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/video-playlists?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf", singletonList(PeertubeSearchQueryHandlerFactory.PLAYLISTS), "").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/video-playlists?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans", singletonList(PeertubeSearchQueryHandlerFactory.PLAYLISTS), "").getUrl());
}
@Test
void testChannelSearch() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/video-channels?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf", singletonList(PeertubeSearchQueryHandlerFactory.CHANNELS), "").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/video-channels?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans", singletonList(PeertubeSearchQueryHandlerFactory.CHANNELS), "").getUrl());
}
}