CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #321 from wb9688/fix-nanojson

Browse source 
Return null instead of "" in getTextFromObject()
This commit is contained in:
Tobias Groza 2020-05-02 15:56:28 +02:00 committed by GitHub
commit 2780e716b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 65 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,11 +17,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 25.07.16.
@ -306,10 +310,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ContentNotSupportedException("This channel has no Videos tab");
}
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos.")) {
final String messageRendererText = getTextFromObject(videoTab.getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("messageRenderer").getObject("text"));
if (messageRendererText != null
&& messageRendererText.equals("This channel has no videos.")) {
return null;
}

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View file

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -19,13 +20,14 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@ -249,7 +251,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
@ -259,7 +261,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public long getDuration() throws ParsingException {
final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!duration.isEmpty()) {
if (duration != null && !duration.isEmpty()) {
return YoutubeParsingHelper.parseDurationString(duration);
}
throw new ParsingException("Could not get duration");
@ -269,7 +271,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public String getUploaderName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get uploader name");
@ -288,11 +290,13 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return null;
} else {
final JsonObject navigationEndpoint = info.getArray("flexColumns")
final JsonObject navigationEndpointHolder = info.getArray("flexColumns")
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint");
.getObject("text").getArray("runs").getObject(0);
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
if (!navigationEndpointHolder.has("navigationEndpoint")) return null;
final String url = getUrlFromNavigationEndpoint(navigationEndpointHolder.getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
return url;
@ -319,7 +323,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!viewCount.isEmpty()) {
if (viewCount != null && !viewCount.isEmpty()) {
return Utils.mixedNumberWordToLong(viewCount);
}
throw new ParsingException("Could not get view count");
@ -359,7 +363,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
@ -378,7 +382,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public long getSubscriberCount() throws ParsingException {
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!viewCount.isEmpty()) {
if (viewCount != null && !viewCount.isEmpty()) {
return Utils.mixedNumberWordToLong(viewCount);
}
throw new ParsingException("Could not get subscriber count");
@ -414,7 +418,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
@ -439,7 +443,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
}
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get uploader name");
@ -452,7 +456,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!count.isEmpty()) {
if (count != null && !count.isEmpty()) {
if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100;
} else {

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -14,9 +15,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
@ -81,7 +83,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(playlistInfo.getObject("title"));
if (!name.isEmpty()) return name;
if (name != null && !name.isEmpty()) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
@ -35,8 +36,6 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
@ -50,6 +49,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
@ -117,7 +119,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
String title = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("title"));
if (title.isEmpty()) {
if (title == null || title.isEmpty()) {
title = playerResponse.getObject("videoDetails").getString("title");
if (title == null || title.isEmpty()) throw new ParsingException("Could not get name");
@ -197,7 +199,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
assertPageFetched();
// description with more info on links
String description = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("description"), true);
if (!description.isEmpty()) return new Description(description, Description.HTML);
if (description != null && !description.isEmpty()) return new Description(description, Description.HTML);
// raw non-html description
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
@ -249,7 +251,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String views = getTextFromObject(getVideoPrimaryInfoRenderer().getObject("viewCount")
.getObject("videoViewCountRenderer").getObject("viewCount"));
if (views.isEmpty()) {
if (views == null || views.isEmpty()) {
views = playerResponse.getObject("videoDetails").getString("viewCount");
if (views == null || views.isEmpty()) throw new ParsingException("Could not get view count");
@ -330,7 +332,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
String uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
if (uploaderName.isEmpty()) {
if (uploaderName == null || uploaderName.isEmpty()) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
if (uploaderName == null || uploaderName.isEmpty()) throw new ParsingException("Could not get uploader name");

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View file

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
@ -11,12 +12,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.*;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
/*
@ -93,7 +97,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(videoInfo.getObject("title"));
if (!name.isEmpty()) return name;
if (name != null && !name.isEmpty()) return name;
throw new ParsingException("Could not get name");
}
@ -105,7 +109,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
String duration = getTextFromObject(videoInfo.getObject("lengthText"));
if (duration.isEmpty()) {
if (duration == null || duration.isEmpty()) {
for (Object thumbnailOverlay : videoInfo.getArray("thumbnailOverlays")) {
if (((JsonObject) thumbnailOverlay).has("thumbnailOverlayTimeStatusRenderer")) {
duration = getTextFromObject(((JsonObject) thumbnailOverlay)
@ -113,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
}
if (duration.isEmpty()) throw new ParsingException("Could not get duration");
if (duration == null || duration.isEmpty()) throw new ParsingException("Could not get duration");
}
return YoutubeParsingHelper.parseDurationString(duration);
@ -123,13 +127,13 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
public String getUploaderName() throws ParsingException {
String name = getTextFromObject(videoInfo.getObject("longBylineText"));
if (name.isEmpty()) {
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("ownerText"));
if (name.isEmpty()) {
if (name == null || name.isEmpty()) {
name = getTextFromObject(videoInfo.getObject("shortBylineText"));
if (name.isEmpty()) throw new ParsingException("Could not get uploader name");
if (name == null || name.isEmpty()) throw new ParsingException("Could not get uploader name");
}
}
@ -169,7 +173,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
}
final String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText"));
if (!publishedTimeText.isEmpty()) return publishedTimeText;
if (publishedTimeText != null && !publishedTimeText.isEmpty()) return publishedTimeText;
return null;
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file

@ -22,6 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -32,9 +33,10 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
@ -71,7 +73,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
if (!name.isEmpty()) {
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get Trending name");

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View file

@ -388,11 +388,15 @@ public class YoutubeParsingHelper {
* Get the text from a JSON object that has either a simpleText or a runs array.
* @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or an empty string
* @return text in the JSON object or {@code null}
*/
public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException {
if (textObject == null || textObject.isEmpty()) return null;
if (textObject.has("simpleText")) return textObject.getString("simpleText");
if (textObject.getArray("runs").isEmpty()) return null;
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java
View file

@ -76,7 +76,7 @@ public class MediaCCCStreamExtractorTest {
@Test
public void testUploaderUrl() throws Exception {
assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
assertEquals("https://media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl());
}
@Test