CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #505 from TeamNewPipe/meida.ccc.de_search_fix

Browse source 
[media.ccc.de] Fix NPE in search results if they contain a future talk
This commit is contained in:
Stypox 2021-01-15 14:16:59 +01:00 committed by GitHub
commit 7c774c84ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 36 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java
View file

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -13,9 +12,11 @@ import org.schabi.newpipe.extractor.localization.Localization;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Pattern;
public final class MediaCCCParsingHelper {
private static final Pattern LIVE_STREAM_ID_PATTERN = Pattern.compile("\\w+/\\w+"); // {conference_slug}/{room_slug}
private static JsonArray liveStreams = null;
private MediaCCCParsingHelper() { }
@ -28,12 +29,28 @@ public final class MediaCCCParsingHelper {
}
}
public static boolean isLiveStreamId(final String url) {
final String pattern = "\\w+/\\w+";
return Pattern.matches(pattern, url); // {conference_slug}/{room_slug}
/**
* Check whether an id is a live stream id
* @param id the {@code id} to check
* @return returns {@code true} if the {@code id} is formatted like {@code {conference_slug}/{room_slug}};
* {@code false} otherwise
*/
public static boolean isLiveStreamId(final String id) {
return LIVE_STREAM_ID_PATTERN.matcher(id).find();
}
public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization) throws ExtractionException {
/**
* Get currently available live streams from
* <a href="https://streaming.media.ccc.de/streams/v2.json">https://streaming.media.ccc.de/streams/v2.json</a>.
* Use this method to cache requests, because they can get quite big.
* TODO: implement better caching policy (max-age: 3 min)
* @param downloader The downloader to use for making the request
* @param localization The localization to be used. Will most likely be ignored.
* @return {@link JsonArray} containing current conferences and info about their rooms and streams.
* @throws ExtractionException if the data could not be fetched or the retrieved data could not be parsed to a {@link JsonArray}
*/
public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization)
throws ExtractionException {
if (liveStreams == null) {
try {
final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View file

@ -81,8 +81,13 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|| getLinkHandler().getContentFilters().isEmpty()) {
JsonArray events = doc.getArray("events");
for (int i = 0; i < events.size(); i++) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
// Ensure only uploaded talks are shown in the search results.
// If the release date is null, the talk has not been held or uploaded yet
// and no streams are going to be available anyway.
if (events.getObject(i).getString("release_date") != null) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
}
}
}
return new InfoItemsPage<>(searchItems, null);
@ -105,7 +110,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json.", jpe);
throw new ExtractionException("Could not parse JSON.", jpe);
}
}
if (getLinkHandler().getContentFilters().contains(CONFERENCES)

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java
View file

@ -55,7 +55,11 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate()));
final String date = getTextualUploadDate();
if (date == null) {
return null; // event is in the future...
}
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
}
@Override

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View file

@ -33,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with querry: " + query, e);
throw new ParsingException("Could not create search string with query: " + query, e);
}
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCLiveStreamListExtractorTest.java
View file

@ -27,7 +27,6 @@ public class MediaCCCLiveStreamListExtractorTest {
@Test
public void getConferencesListTest() throws Exception {
final List<InfoItem> a = extractor.getInitialPage().getItems();
assertTrue(a.size() > 2);
for (int i = 0; i < a.size(); i++) {
final InfoItem b = a.get(i);
assertNotNull(a.get(i).getName());