Extract Video Short Description in YouTube.

In Trending, and Search results.
This commit is contained in:
FireMasterK 2021-09-22 16:19:44 +01:00
parent a9d214478d
commit 94efe86c71
No known key found for this signature in database
GPG key ID: 49451E4482CC5BCD
18 changed files with 1398 additions and 0 deletions

View file

@ -43,6 +43,12 @@ public class BandcampRadioInfoItemExtractor implements StreamInfoItemExtractor {
return BandcampExtractorHelper.parseDate(getTextualUploadDate()); return BandcampExtractorHelper.parseDate(getTextualUploadDate());
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return show.getString("subtitle"); return show.getString("subtitle");

View file

@ -26,6 +26,12 @@ public class BandcampDiscographStreamInfoItemExtractor extends BandcampStreamInf
return null; return null;
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public String getName() { public String getName() {
return discograph.getString("title"); return discograph.getString("title");

View file

@ -60,6 +60,12 @@ public class BandcampPlaylistStreamInfoItemExtractor extends BandcampStreamInfoI
return null; return null;
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
/** /**
* Each track can have its own cover art. Therefore, unless a substitute is provided, * Each track can have its own cover art. Therefore, unless a substitute is provided,
* the thumbnail is extracted using a stream extractor. * the thumbnail is extracted using a stream extractor.

View file

@ -32,6 +32,12 @@ public class BandcampSearchStreamInfoItemExtractor extends BandcampStreamInfoIte
return null; return null;
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return resultInfo.getElementsByClass("heading").text(); return resultInfo.getElementsByClass("heading").text();

View file

@ -95,4 +95,10 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor
public DateWrapper getUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException {
return null; return null;
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
} }

View file

@ -92,4 +92,10 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz")); DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz"));
return new DateWrapper(zonedDateTime.toOffsetDateTime(), false); return new DateWrapper(zonedDateTime.toOffsetDateTime(), false);
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
} }

View file

@ -73,6 +73,12 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date)); return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return event.getString("title"); return event.getString("title");

View file

@ -92,6 +92,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public StreamType getStreamType() { public StreamType getStreamType() {
return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM; return item.getBoolean("isLive") ? StreamType.LIVE_STREAM : StreamType.VIDEO_STREAM;

View file

@ -66,6 +66,12 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public long getViewCount() { public long getViewCount() {
return itemObject.getLong("playback_count"); return itemObject.getLong("playback_count");

View file

@ -77,6 +77,12 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
} }
} }
@Nullable
@Override
public String getShortDescription() {
return null;
}
@Override @Override
public String getName() { public String getName() {
return entryElement.getElementsByTag("title").first().text(); return entryElement.getElementsByTag("title").first().text();

View file

@ -288,4 +288,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\"");
} }
} }
@Nullable
@Override
public String getShortDescription() throws ParsingException {
if(videoInfo.has("detailedMetadataSnippets"))
return getTextFromObject(videoInfo.getArray("detailedMetadataSnippets").getObject(0).getObject("snippetText"));
if(videoInfo.has("descriptionSnippet"))
return getTextFromObject(videoInfo.getObject("descriptionSnippet"));
return null;
}
} }

View file

@ -32,6 +32,7 @@ public class StreamInfoItem extends InfoItem {
private final StreamType streamType; private final StreamType streamType;
private String uploaderName; private String uploaderName;
private String shortDescription;
private String textualUploadDate; private String textualUploadDate;
@Nullable @Nullable
private DateWrapper uploadDate; private DateWrapper uploadDate;
@ -92,6 +93,14 @@ public class StreamInfoItem extends InfoItem {
this.uploaderAvatarUrl = uploaderAvatarUrl; this.uploaderAvatarUrl = uploaderAvatarUrl;
} }
public String getShortDescription() {
return shortDescription;
}
public void setShortDescription(String shortDescription) {
this.shortDescription = shortDescription;
}
@Nullable @Nullable
public String getTextualUploadDate() { public String getTextualUploadDate() {
return textualUploadDate; return textualUploadDate;

View file

@ -116,4 +116,14 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
@Nullable @Nullable
DateWrapper getUploadDate() throws ParsingException; DateWrapper getUploadDate() throws ParsingException;
/**
* Get the video's short description.
*
* @return The video's short description or {@code null} if not provided by the service.
* @throws ParsingException if there is an error in the extraction
*/
@Nullable
String getShortDescription() throws ParsingException;
} }

View file

@ -101,6 +101,11 @@ public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem,
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try {
resultItem.setShortDescription(extractor.getShortDescription());
} catch (Exception e) {
addError(e);
}
return resultItem; return resultItem;
} }

View file

@ -343,4 +343,34 @@ public class YoutubeSearchExtractorTest {
} }
} }
} }
public static class VideoDescription extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "44wLAzydRFU";
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "video_description"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@Test
public void testVideoDescription() throws IOException, ExtractionException {
final List<InfoItem> items = extractor.getInitialPage().getItems();
assertNotNull(((StreamInfoItem) items.get(0)).getShortDescription());
}
}
} }