mirror of
https://github.com/TeamPiped/Piped-Backend.git
synced 2024-08-14 23:51:41 +00:00
Merge pull request #586 from TeamPiped/npe-update
Update NewPipeExtractor
This commit is contained in:
commit
5d085ce592
7 changed files with 100 additions and 53 deletions
|
@ -17,7 +17,7 @@ dependencies {
|
||||||
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
|
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
|
||||||
implementation 'commons-codec:commons-codec:1.15'
|
implementation 'commons-codec:commons-codec:1.15'
|
||||||
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
|
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
|
||||||
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:a0037b718448d744b151412953ed28254acf5389'
|
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:251b204a23742c3413426c9b6131d084b190636f'
|
||||||
implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
|
implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
|
||||||
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.1'
|
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.1'
|
||||||
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.1'
|
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.1'
|
||||||
|
|
|
@ -11,12 +11,14 @@ import me.kavin.piped.utils.obj.federation.FederatedVideoInfo;
|
||||||
import me.kavin.piped.utils.resp.InvalidRequestResponse;
|
import me.kavin.piped.utils.resp.InvalidRequestResponse;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.hibernate.StatelessSession;
|
import org.hibernate.StatelessSession;
|
||||||
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
import org.schabi.newpipe.extractor.ListExtractor;
|
import org.schabi.newpipe.extractor.ListExtractor;
|
||||||
import org.schabi.newpipe.extractor.Page;
|
import org.schabi.newpipe.extractor.Page;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfo;
|
import org.schabi.newpipe.extractor.channel.ChannelInfo;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
|
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
import org.schabi.newpipe.extractor.exceptions.ParsingException;
|
||||||
|
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
|
||||||
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||||
|
|
||||||
|
@ -28,6 +30,7 @@ import java.util.stream.Collectors;
|
||||||
|
|
||||||
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
||||||
import static me.kavin.piped.consts.Constants.mapper;
|
import static me.kavin.piped.consts.Constants.mapper;
|
||||||
|
import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs;
|
||||||
import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
|
import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
|
||||||
import static me.kavin.piped.utils.URLUtils.rewriteURL;
|
import static me.kavin.piped.utils.URLUtils.rewriteURL;
|
||||||
|
|
||||||
|
@ -38,9 +41,14 @@ public class ChannelHandlers {
|
||||||
|
|
||||||
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
|
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
|
||||||
|
|
||||||
final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems());
|
final ChannelTabInfo tabInfo = ChannelTabInfo.getInfo(YOUTUBE_SERVICE, collectPreloadedTabs(info.getTabs()).get(0));
|
||||||
|
|
||||||
Multithreading.runAsync(() -> info.getRelatedItems().forEach(infoItem -> {
|
final List<ContentItem> relatedStreams = collectRelatedItems(tabInfo.getRelatedItems());
|
||||||
|
|
||||||
|
Multithreading.runAsync(() -> tabInfo.getRelatedItems()
|
||||||
|
.stream().filter(StreamInfoItem.class::isInstance)
|
||||||
|
.map(StreamInfoItem.class::cast)
|
||||||
|
.forEach(infoItem -> {
|
||||||
if (
|
if (
|
||||||
infoItem.getUploadDate() != null &&
|
infoItem.getUploadDate() != null &&
|
||||||
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||||
|
@ -55,7 +63,8 @@ public class ChannelHandlers {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
}));
|
})
|
||||||
|
);
|
||||||
|
|
||||||
Multithreading.runAsync(() -> {
|
Multithreading.runAsync(() -> {
|
||||||
try {
|
try {
|
||||||
|
@ -75,8 +84,10 @@ public class ChannelHandlers {
|
||||||
|
|
||||||
ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified());
|
ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified());
|
||||||
|
|
||||||
Set<String> ids = info.getRelatedItems()
|
Set<String> ids = tabInfo.getRelatedItems()
|
||||||
.stream()
|
.stream()
|
||||||
|
.filter(StreamInfoItem.class::isInstance)
|
||||||
|
.map(StreamInfoItem.class::cast)
|
||||||
.filter(item -> {
|
.filter(item -> {
|
||||||
long time = item.getUploadDate() != null
|
long time = item.getUploadDate() != null
|
||||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||||
|
@ -94,7 +105,10 @@ public class ChannelHandlers {
|
||||||
|
|
||||||
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
|
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
|
||||||
|
|
||||||
for (StreamInfoItem item : info.getRelatedItems()) {
|
tabInfo.getRelatedItems()
|
||||||
|
.stream()
|
||||||
|
.filter(StreamInfoItem.class::isInstance)
|
||||||
|
.map(StreamInfoItem.class::cast).forEach(item -> {
|
||||||
long time = item.getUploadDate() != null
|
long time = item.getUploadDate() != null
|
||||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||||
: System.currentTimeMillis();
|
: System.currentTimeMillis();
|
||||||
|
@ -112,19 +126,20 @@ public class ChannelHandlers {
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
ExceptionHandler.handle(e);
|
ExceptionHandler.handle(e);
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
String nextpage = null;
|
String nextpage = null;
|
||||||
if (info.hasNextPage()) {
|
if (tabInfo.hasNextPage()) {
|
||||||
Page page = info.getNextPage();
|
Page page = tabInfo.getNextPage();
|
||||||
nextpage = mapper.writeValueAsString(page);
|
nextpage = mapper.writeValueAsString(page);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<ChannelTab> tabs = info.getTabs()
|
List<ChannelTab> tabs = info.getTabs()
|
||||||
.stream()
|
.stream()
|
||||||
|
.filter(tab -> !tab.getContentFilters().contains(ChannelTabs.VIDEOS))
|
||||||
.map(tab -> {
|
.map(tab -> {
|
||||||
try {
|
try {
|
||||||
return new ChannelTab(tab.getContentFilters().get(0), mapper.writeValueAsString(tab));
|
return new ChannelTab(tab.getContentFilters().get(0), mapper.writeValueAsString(tab));
|
||||||
|
@ -152,8 +167,11 @@ public class ChannelHandlers {
|
||||||
if (prevpage == null)
|
if (prevpage == null)
|
||||||
ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter"));
|
ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter"));
|
||||||
|
|
||||||
ListExtractor.InfoItemsPage<StreamInfoItem> info = ChannelInfo.getMoreItems(YOUTUBE_SERVICE,
|
String channelUrl = "https://youtube.com/channel/" + channelId;
|
||||||
"https://youtube.com/channel/" + channelId, prevpage);
|
String url = channelUrl + "/videos";
|
||||||
|
|
||||||
|
ListExtractor.InfoItemsPage<InfoItem> info = ChannelTabInfo.getMoreItems(YOUTUBE_SERVICE,
|
||||||
|
new ListLinkHandler(url, url, channelId, List.of("videos"), ""), prevpage);
|
||||||
|
|
||||||
final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems());
|
final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems());
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@ public class PlaylistHandlers {
|
||||||
}
|
}
|
||||||
|
|
||||||
final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()),
|
final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()),
|
||||||
rewriteURL(info.getBannerUrl()), nextpage,
|
info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage,
|
||||||
info.getUploaderName().isEmpty() ? null : info.getUploaderName(),
|
info.getUploaderName().isEmpty() ? null : info.getUploaderName(),
|
||||||
substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()),
|
substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()),
|
||||||
(int) info.getStreamCount(), relatedStreams);
|
(int) info.getStreamCount(), relatedStreams);
|
||||||
|
|
|
@ -67,7 +67,7 @@ public class AuthPlaylistHandlers {
|
||||||
video.getDuration(), -1, -1, channel.isVerified(), false));
|
video.getDuration(), -1, -1, channel.isVerified(), false));
|
||||||
}
|
}
|
||||||
|
|
||||||
final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), null, null, pl.getOwner().getUsername(),
|
final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), pl.getShortDescription(), null, null, pl.getOwner().getUsername(),
|
||||||
null, null, videos.size(), relatedStreams);
|
null, null, videos.size(), relatedStreams);
|
||||||
|
|
||||||
return mapper.writeValueAsBytes(playlist);
|
return mapper.writeValueAsBytes(playlist);
|
||||||
|
|
|
@ -4,6 +4,9 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
|
||||||
import me.kavin.piped.utils.obj.*;
|
import me.kavin.piped.utils.obj.*;
|
||||||
import org.schabi.newpipe.extractor.InfoItem;
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
|
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
|
||||||
|
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
|
||||||
|
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
|
||||||
|
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
|
||||||
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
|
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
import org.schabi.newpipe.extractor.stream.StreamInfo;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||||
|
@ -13,6 +16,7 @@ import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
||||||
import static me.kavin.piped.utils.URLUtils.*;
|
import static me.kavin.piped.utils.URLUtils.*;
|
||||||
|
|
||||||
public class CollectionUtils {
|
public class CollectionUtils {
|
||||||
|
@ -122,4 +126,12 @@ public class CollectionUtils {
|
||||||
item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
|
item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
|
||||||
item.isVerified());
|
item.isVerified());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static List<ReadyChannelTabListLinkHandler> collectPreloadedTabs(List<ListLinkHandler> tabs) {
|
||||||
|
return tabs
|
||||||
|
.stream()
|
||||||
|
.filter(ReadyChannelTabListLinkHandler.class::isInstance)
|
||||||
|
.map(ReadyChannelTabListLinkHandler.class::cast)
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,9 @@ import me.kavin.piped.utils.obj.db.*;
|
||||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||||
import org.hibernate.SharedSessionContract;
|
import org.hibernate.SharedSessionContract;
|
||||||
import org.hibernate.StatelessSession;
|
import org.hibernate.StatelessSession;
|
||||||
|
import org.schabi.newpipe.extractor.InfoItem;
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfo;
|
import org.schabi.newpipe.extractor.channel.ChannelInfo;
|
||||||
|
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
|
||||||
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
|
||||||
|
|
||||||
|
@ -19,6 +21,8 @@ import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
|
||||||
|
|
||||||
public class DatabaseHelper {
|
public class DatabaseHelper {
|
||||||
|
|
||||||
public static User getUserFromSession(String session) {
|
public static User getUserFromSession(String session) {
|
||||||
|
@ -207,13 +211,25 @@ public class DatabaseHelper {
|
||||||
});
|
});
|
||||||
|
|
||||||
Multithreading.runAsync(() -> {
|
Multithreading.runAsync(() -> {
|
||||||
for (StreamInfoItem item : info.getRelatedItems()) {
|
CollectionUtils.collectPreloadedTabs(info.getTabs())
|
||||||
|
.stream()
|
||||||
|
.parallel()
|
||||||
|
.map(tab -> {
|
||||||
|
try {
|
||||||
|
return ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab).getRelatedItems();
|
||||||
|
} catch (ExtractionException | IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter(StreamInfoItem.class::isInstance)
|
||||||
|
.map(StreamInfoItem.class::cast)
|
||||||
|
.forEach(item -> {
|
||||||
long time = item.getUploadDate() != null
|
long time = item.getUploadDate() != null
|
||||||
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
|
||||||
: System.currentTimeMillis();
|
: System.currentTimeMillis();
|
||||||
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
|
||||||
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
|
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
|
||||||
}
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
return channel;
|
return channel;
|
||||||
|
|
|
@ -4,14 +4,15 @@ import java.util.List;
|
||||||
|
|
||||||
public class Playlist {
|
public class Playlist {
|
||||||
|
|
||||||
public String name, thumbnailUrl, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar;
|
public String name, thumbnailUrl, description, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar;
|
||||||
public int videos;
|
public int videos;
|
||||||
public List<ContentItem> relatedStreams;
|
public List<ContentItem> relatedStreams;
|
||||||
|
|
||||||
public Playlist(String name, String thumbnailUrl, String bannerUrl, String nextpage, String uploader,
|
public Playlist(String name, String thumbnailUrl, String description, String bannerUrl, String nextpage, String uploader,
|
||||||
String uploaderUrl, String uploaderAvatar, int videos, List<ContentItem> relatedStreams) {
|
String uploaderUrl, String uploaderAvatar, int videos, List<ContentItem> relatedStreams) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.thumbnailUrl = thumbnailUrl;
|
this.thumbnailUrl = thumbnailUrl;
|
||||||
|
this.description = description;
|
||||||
this.bannerUrl = bannerUrl;
|
this.bannerUrl = bannerUrl;
|
||||||
this.nextpage = nextpage;
|
this.nextpage = nextpage;
|
||||||
this.videos = videos;
|
this.videos = videos;
|
||||||
|
|
Loading…
Reference in a new issue