From 9527ad4133405e8174d227d548485b7d94bee74d Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 25 Nov 2020 10:56:25 +0530 Subject: [PATCH] Channel Pages and search suggestions. --- src/main/java/me/kavin/piped/Main.java | 31 +++++++- .../me/kavin/piped/utils/ResponseHelper.java | 77 ++++++++++++++++--- .../me/kavin/piped/utils/obj/Channel.java | 9 ++- .../me/kavin/piped/utils/obj/ChannelPage.java | 22 ++++++ .../obj/{Stream.java => PipedStream.java} | 4 +- .../me/kavin/piped/utils/obj/Streams.java | 8 +- 6 files changed, 131 insertions(+), 20 deletions(-) create mode 100644 src/main/java/me/kavin/piped/utils/obj/ChannelPage.java rename src/main/java/me/kavin/piped/utils/obj/{Stream.java => PipedStream.java} (74%) diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index 82cffca..73286e2 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -20,8 +20,6 @@ public class Main { public static void main(String[] args) throws Exception { - System.setProperty("file.encoding", "UTF-8"); - // SyndFeed feed = new SyndFeedInput().build(new XmlReader(new FileInputStream("pubsub.xml"))); // // feed.getEntries().forEach(entry -> { @@ -97,6 +95,35 @@ public class Main { }); + routes.get("/nextpage/channels/{channelId}", (req, res) -> { + + QueryStringDecoder query = new QueryStringDecoder(req.uri()); + + try { + return writeResponse(res, ResponseHelper.channelPageResponse(req.param("channelId"), + query.parameters().get("url").get(0)), 200, "public, max-age=3600"); + } catch (Exception e) { + e.printStackTrace(); + return writeResponse(res, ExceptionUtils.getStackTrace(e), 500, "private"); + } + + }); + + routes.get("/suggestions", (req, res) -> { + + QueryStringDecoder query = new QueryStringDecoder(req.uri()); + + try { + return writeResponse(res, + ResponseHelper.suggestionsResponse(query.parameters().get("query").get(0)), 200, + "public, max-age=600"); + } catch (Exception e) { + e.printStackTrace(); + return writeResponse(res, ExceptionUtils.getStackTrace(e), 500, "private"); + } + + }); + routes.get("/trending", (req, res) -> { try { diff --git a/src/main/java/me/kavin/piped/utils/ResponseHelper.java b/src/main/java/me/kavin/piped/utils/ResponseHelper.java index 3d2ea3a..3e719f3 100644 --- a/src/main/java/me/kavin/piped/utils/ResponseHelper.java +++ b/src/main/java/me/kavin/piped/utils/ResponseHelper.java @@ -11,23 +11,29 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.json.JSONObject; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.channel.ChannelInfo; import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskInfo; +import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.Channel; -import me.kavin.piped.utils.obj.Stream; +import me.kavin.piped.utils.obj.ChannelPage; +import me.kavin.piped.utils.obj.PipedStream; import me.kavin.piped.utils.obj.StreamItem; import me.kavin.piped.utils.obj.Streams; import me.kavin.piped.utils.obj.Subtitle; @@ -65,13 +71,13 @@ public class ResponseHelper { info.getSubtitles().forEach(subtitle -> subtitles .add(new Subtitle(rewriteURL(subtitle.getUrl()), subtitle.getFormat().getMimeType()))); - final List videoStreams = new ObjectArrayList<>(); - final List audioStreams = new ObjectArrayList<>(); + final List videoStreams = new ObjectArrayList<>(); + final List audioStreams = new ObjectArrayList<>(); final String lbryURL = futureLBRY.get(); if (lbryURL != null) - videoStreams.add(new Stream(lbryURL, "MP4", "LBRY", "video/mp4")); + videoStreams.add(new PipedStream(lbryURL, "MP4", "LBRY", "video/mp4")); String hls = null; boolean livestream = false; @@ -79,14 +85,32 @@ public class ResponseHelper { if ((hls = info.getHlsUrl()) != null && !hls.isEmpty()) livestream = true; - info.getVideoOnlyStreams().forEach(stream -> videoStreams.add(new Stream(rewriteURL(stream.getUrl()), + long minexpire = Long.MAX_VALUE; + + ObjectArrayList allStreams = new ObjectArrayList<>(); + + allStreams.addAll(info.getVideoStreams()); + allStreams.addAll(info.getAudioStreams()); + allStreams.addAll(info.getVideoOnlyStreams()); + + for (Stream stream : allStreams) { + + long expire = Long.parseLong(StringUtils.substringBetween(stream.getUrl(), "expire=", "&")); + + if (expire < minexpire) + minexpire = expire; + + } + + info.getVideoOnlyStreams().forEach(stream -> videoStreams.add(new PipedStream(rewriteURL(stream.getUrl()), String.valueOf(stream.getFormat()), stream.getResolution(), stream.getFormat().getMimeType()))); - info.getVideoStreams().forEach(stream -> videoStreams.add(new Stream(rewriteURL(stream.getUrl()), + info.getVideoStreams().forEach(stream -> videoStreams.add(new PipedStream(rewriteURL(stream.getUrl()), String.valueOf(stream.getFormat()), stream.getResolution(), stream.getFormat().getMimeType()))); - info.getAudioStreams().forEach( - stream -> audioStreams.add(new Stream(rewriteURL(stream.getUrl()), String.valueOf(stream.getFormat()), - stream.getAverageBitrate() + " kbps", stream.getFormat().getMimeType()))); + info.getAudioStreams() + .forEach(stream -> audioStreams + .add(new PipedStream(rewriteURL(stream.getUrl()), String.valueOf(stream.getFormat()), + stream.getAverageBitrate() + " kbps", stream.getFormat().getMimeType()))); final List relatedStreams = new ObjectArrayList<>(); @@ -121,13 +145,38 @@ public class ResponseHelper { item.getDuration(), item.getViewCount())); }); + String nextpage = info.hasNextPage() ? info.getNextPage().getUrl() : null; + final Channel channel = new Channel(info.getName(), info.getAvatarUrl(), info.getBannerUrl(), - info.getDescription(), relatedStreams); + info.getDescription(), nextpage, relatedStreams); return Constants.mapper.writeValueAsString(channel); } + public static final String channelPageResponse(String channelId, String url) + throws IOException, ExtractionException, InterruptedException { + + InfoItemsPage page = ChannelInfo.getMoreItems(Constants.YOUTUBE_SERVICE, + "https://youtube.com/channel/" + channelId, new Page(url)); + + final List relatedStreams = new ObjectArrayList<>(); + + page.getItems().forEach(o -> { + StreamInfoItem item = o; + relatedStreams.add(new StreamItem(item.getUrl().substring(23), item.getName(), + rewriteURL(item.getThumbnailUrl()), item.getUploaderName(), item.getUploaderUrl().substring(23), + item.getDuration(), item.getViewCount())); + }); + + String nextpage = page.hasNextPage() ? page.getNextPage().getUrl() : null; + + final ChannelPage channelpage = new ChannelPage(nextpage, relatedStreams); + + return Constants.mapper.writeValueAsString(channelpage); + + } + final List relatedStreams = new ObjectArrayList<>(); public static final String trendingResponse() throws ParsingException, ExtractionException, IOException { @@ -148,6 +197,14 @@ public class ResponseHelper { return Constants.mapper.writeValueAsString(relatedStreams); } + public static final String suggestionsResponse(String query) + throws JsonProcessingException, IOException, ExtractionException { + + return Constants.mapper + .writeValueAsString(Constants.YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)); + + } + private static final String getLBRYStreamURL(String videoId) throws IOException, InterruptedException { String lbryId = new JSONObject(Constants.h2client.send(HttpRequest diff --git a/src/main/java/me/kavin/piped/utils/obj/Channel.java b/src/main/java/me/kavin/piped/utils/obj/Channel.java index eb0900e..3041bfb 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Channel.java +++ b/src/main/java/me/kavin/piped/utils/obj/Channel.java @@ -4,14 +4,15 @@ import java.util.List; public class Channel { - private String name, avatarUrl, bannerUrl, description; + private String name, avatarUrl, bannerUrl, description, nextpage; private List relatedStreams; - public Channel(String name, String avatarUrl, String bannerUrl, String description, + public Channel(String name, String avatarUrl, String bannerUrl, String description, String nextpage, List relatedStreams) { this.name = name; this.avatarUrl = avatarUrl; this.description = description; + this.nextpage = nextpage; this.relatedStreams = relatedStreams; } @@ -31,6 +32,10 @@ public class Channel { return description; } + public String getNextpage() { + return nextpage; + } + public List getRelatedStreams() { return relatedStreams; } diff --git a/src/main/java/me/kavin/piped/utils/obj/ChannelPage.java b/src/main/java/me/kavin/piped/utils/obj/ChannelPage.java new file mode 100644 index 0000000..802ddb8 --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/obj/ChannelPage.java @@ -0,0 +1,22 @@ +package me.kavin.piped.utils.obj; + +import java.util.List; + +public class ChannelPage { + + private String nextpage; + private List relatedStreams; + + public ChannelPage(String nextpage, List relatedStreams) { + this.nextpage = nextpage; + this.relatedStreams = relatedStreams; + } + + public String getNextpage() { + return nextpage; + } + + public List getRelatedStreams() { + return relatedStreams; + } +} diff --git a/src/main/java/me/kavin/piped/utils/obj/Stream.java b/src/main/java/me/kavin/piped/utils/obj/PipedStream.java similarity index 74% rename from src/main/java/me/kavin/piped/utils/obj/Stream.java rename to src/main/java/me/kavin/piped/utils/obj/PipedStream.java index 1676ae1..db5e05c 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Stream.java +++ b/src/main/java/me/kavin/piped/utils/obj/PipedStream.java @@ -1,10 +1,10 @@ package me.kavin.piped.utils.obj; -public class Stream { +public class PipedStream { private String url, format, quality, mimeType; - public Stream(String url, String format, String quality, String mimeType) { + public PipedStream(String url, String format, String quality, String mimeType) { this.url = url; this.format = format; this.quality = quality; diff --git a/src/main/java/me/kavin/piped/utils/obj/Streams.java b/src/main/java/me/kavin/piped/utils/obj/Streams.java index 941e92b..ce14713 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Streams.java +++ b/src/main/java/me/kavin/piped/utils/obj/Streams.java @@ -8,7 +8,7 @@ public class Streams { private long duration, views, likes, dislikes; - private List audioStreams, videoStreams; + private List audioStreams, videoStreams; private List relatedStreams; @@ -18,7 +18,7 @@ public class Streams { public Streams(String title, String description, String uploadDate, String uploader, String uploaderUrl, String uploaderAvatar, String thumbnailUrl, long duration, long views, long likes, long dislikes, - List audioStreams, List videoStreams, List relatedStreams, + List audioStreams, List videoStreams, List relatedStreams, List subtitles, boolean livestream, String hls) { this.title = title; this.description = description; @@ -83,11 +83,11 @@ public class Streams { return dislikes; } - public List getAudioStreams() { + public List getAudioStreams() { return audioStreams; } - public List getVideoStreams() { + public List getVideoStreams() { return videoStreams; }