CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #495 from B0pol/sepiasearch

Browse source 
[PeerTube] Implement sepia search
This commit is contained in:
Tobias Groza 2021-01-06 11:06:36 +01:00 committed by GitHub
commit ab62464e86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 107 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View file

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
@ -63,6 +64,19 @@ public class PeertubeParsingHelper {
}
public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl) throws ParsingException {
collectStreamsFrom(collector, json, baseUrl, false);
}
/**
* Collect stream from json with collector
*
* @param collector the collector used to collect information
* @param json the file to retrieve data from
* @param baseUrl the base Url of the instance
* @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor
* @throws ParsingException
*/
public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl, boolean sepia) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
@ -73,9 +87,15 @@ public class PeertubeParsingHelper {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
PeertubeStreamInfoItemExtractor extractor;
if (sepia) {
extractor = new PeertubeSepiaStreamInfoItemExtractor(item, baseUrl);
} else {
extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
}
collector.commit(extractor);
}
}
}
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View file

@ -15,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
@ -59,7 +61,12 @@ public class PeertubeService extends StreamingService {
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new PeertubeSearchExtractor(this, queryHandler);
final List<String> contentFilters = queryHandler.getContentFilters();
boolean external = false;
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
external = true;
}
return new PeertubeSearchExtractor(this, queryHandler, external);
}
@Override

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View file

@ -30,8 +30,17 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor {
// if we should use PeertubeSepiaStreamInfoItemExtractor
private boolean sepia;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
this(service, linkHandler, false);
}
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, boolean sepia) {
super(service, linkHandler);
this.sepia = sepia;
}
@Nonnull
@ -79,7 +88,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
final long total = json.getLong("total");
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
collectStreamsFrom(collector, json, getBaseUrl(), sepia);
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSepiaStreamInfoItemExtractor.java Normal file
View file

@ -0,0 +1,22 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
/**
* A StreamInfoItem collected from SepiaSearch
*/
public class PeertubeSepiaStreamInfoItemExtractor extends PeertubeStreamInfoItemExtractor {
public PeertubeSepiaStreamInfoItemExtractor(JsonObject item, String baseUrl) {
super(item, baseUrl);
final String embedUrl = super.item.getString("embedUrl");
final String embedPath = super.item.getString("embedPath");
final String itemBaseUrl = embedUrl.replace(embedPath, "");
setBaseUrl(itemBaseUrl);
// Usually, all videos, pictures and other content are hosted on the instance,
// or can be accessed by the same URL path if the instance with baseUrl federates the one where the video is actually uploaded
// But it can't be accessed with Sepiasearch, so we use the item's instance as base URL
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View file

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -11,8 +10,9 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject item;
private final String baseUrl;
private String baseUrl;
public PeertubeStreamInfoItemExtractor(final JsonObject item, final String baseUrl) {
this.item = item;
@ -84,4 +84,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
public long getDuration() {
return item.getLong("duration");
}
protected void setBaseUrl(final String baseUrl) {
this.baseUrl = baseUrl;
}
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View file

@ -12,6 +12,8 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
public static final String SEARCH_ENDPOINT = "/api/v1/search/videos";
public static PeertubeSearchQueryHandlerFactory getInstance() {
@ -20,7 +22,12 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
String baseUrl;
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
baseUrl = SEPIA_BASE_URL;
} else {
baseUrl = ServiceList.PeerTube.getBaseUrl();
}
return getUrl(searchString, contentFilters, sortFilter, baseUrl);
}
@ -38,6 +45,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String[] getAvailableContentFilter() {
return new String[]{VIDEOS};
return new String[]{
VIDEOS,
SEPIA_VIDEOS
};
}
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java
View file

@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import javax.annotation.Nullable;
@ -43,6 +44,29 @@ public class PeertubeSearchExtractorTest {
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
public static class SepiaSearch extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "kde";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = PeerTube.getSearchExtractor(QUERY, singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return PeerTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
public static class PagingTest {
@Test
public void duplicatedItemsCheck() throws Exception {

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchQHTest.java
View file

@ -3,7 +3,9 @@ package org.schabi.newpipe.extractor.services.peertube.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
@ -22,5 +24,7 @@ public class PeertubeSearchQHTest {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://sepiasearch.org/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "").getUrl());
assertEquals("https://anotherpeertubeindex.com/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "", "https://anotherpeertubeindex.com").getUrl());
}
}