TeamPiped-mirror
/
Piped-Backend
mirror of https://github.com/TeamPiped/Piped-Backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #586 from TeamPiped/npe-update 

Update NewPipeExtractor
This commit is contained in:
Kavin 2023-05-22 21:12:43 +01:00 committed by GitHub
parent 8b9973e874 fc7d3dd2a5
commit 5d085ce592
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 100 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -17,7 +17,7 @@ dependencies {
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
implementation 'commons-codec:commons-codec:1.15'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:a0037b718448d744b151412953ed28254acf5389'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:251b204a23742c3413426c9b6131d084b190636f'
implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.1'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.1'

100
src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java
View File

@ -11,12 +11,14 @@ import me.kavin.piped.utils.obj.federation.FederatedVideoInfo;
import me.kavin.piped.utils.resp.InvalidRequestResponse;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -28,6 +30,7 @@ import java.util.stream.Collectors;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs;
import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
import static me.kavin.piped.utils.URLUtils.rewriteURL;
@ -38,24 +41,30 @@ public class ChannelHandlers {
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems());
final ChannelTabInfo tabInfo = ChannelTabInfo.getInfo(YOUTUBE_SERVICE, collectPreloadedTabs(info.getTabs()).get(0));
Multithreading.runAsync(() -> info.getRelatedItems().forEach(infoItem -> {
if (
infoItem.getUploadDate() != null &&
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
< TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)
)
try {
MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo(
StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24),
infoItem.getName(),
infoItem.getDuration(), infoItem.getViewCount())
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}));
final List<ContentItem> relatedStreams = collectRelatedItems(tabInfo.getRelatedItems());
Multithreading.runAsync(() -> tabInfo.getRelatedItems()
.stream().filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.forEach(infoItem -> {
if (
infoItem.getUploadDate() != null &&
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
< TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)
)
try {
MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo(
StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24),
infoItem.getName(),
infoItem.getDuration(), infoItem.getViewCount())
);
} catch (IOException e) {
throw new RuntimeException(e);
}
})
);
Multithreading.runAsync(() -> {
try {
@ -75,8 +84,10 @@ public class ChannelHandlers {
ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified());
Set<String> ids = info.getRelatedItems()
Set<String> ids = tabInfo.getRelatedItems()
.stream()
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.filter(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
@ -94,37 +105,41 @@ public class ChannelHandlers {
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
try {
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
var video = videos.stream()
.filter(v -> v.getId().equals(id))
.findFirst();
if (video.isPresent()) {
VideoHelpers.updateVideo(id, item);
} else {
VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
}
tabInfo.getRelatedItems()
.stream()
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast).forEach(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
try {
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
var video = videos.stream()
.filter(v -> v.getId().equals(id))
.findFirst();
if (video.isPresent()) {
VideoHelpers.updateVideo(id, item);
} else {
VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
});
}
}
});
String nextpage = null;
if (info.hasNextPage()) {
Page page = info.getNextPage();
if (tabInfo.hasNextPage()) {
Page page = tabInfo.getNextPage();
nextpage = mapper.writeValueAsString(page);
}
List<ChannelTab> tabs = info.getTabs()
.stream()
.filter(tab -> !tab.getContentFilters().contains(ChannelTabs.VIDEOS))
.map(tab -> {
try {
return new ChannelTab(tab.getContentFilters().get(0), mapper.writeValueAsString(tab));
@ -152,8 +167,11 @@ public class ChannelHandlers {
if (prevpage == null)
ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter"));
ListExtractor.InfoItemsPage<StreamInfoItem> info = ChannelInfo.getMoreItems(YOUTUBE_SERVICE,
"https://youtube.com/channel/" + channelId, prevpage);
String channelUrl = "https://youtube.com/channel/" + channelId;
String url = channelUrl + "/videos";
ListExtractor.InfoItemsPage<InfoItem> info = ChannelTabInfo.getMoreItems(YOUTUBE_SERVICE,
new ListLinkHandler(url, url, channelId, List.of("videos"), ""), prevpage);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems());

2
src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java
View File

@ -61,7 +61,7 @@ public class PlaylistHandlers {
}
final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()),
rewriteURL(info.getBannerUrl()), nextpage,
info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage,
info.getUploaderName().isEmpty() ? null : info.getUploaderName(),
substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()),
(int) info.getStreamCount(), relatedStreams);

2
src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java
View File

@ -67,7 +67,7 @@ public class AuthPlaylistHandlers {
video.getDuration(), -1, -1, channel.isVerified(), false));
}
final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), null, null, pl.getOwner().getUsername(),
final Playlist playlist = new Playlist(pl.getName(), rewriteURL(pl.getThumbnail()), pl.getShortDescription(), null, null, pl.getOwner().getUsername(),
null, null, videos.size(), relatedStreams);
return mapper.writeValueAsBytes(playlist);

12
src/main/java/me/kavin/piped/utils/CollectionUtils.java
View File

@ -4,6 +4,9 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.kavin.piped.utils.obj.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -13,6 +16,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Optional;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.utils.URLUtils.*;
public class CollectionUtils {
@ -122,4 +126,12 @@ public class CollectionUtils {
item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
item.isVerified());
}
public static List<ReadyChannelTabListLinkHandler> collectPreloadedTabs(List<ListLinkHandler> tabs) {
return tabs
.stream()
.filter(ReadyChannelTabListLinkHandler.class::isInstance)
.map(ReadyChannelTabListLinkHandler.class::cast)
.toList();
}
}

30
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View File

@ -9,7 +9,9 @@ import me.kavin.piped.utils.obj.db.*;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hibernate.SharedSessionContract;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -19,6 +21,8 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
public class DatabaseHelper {
public static User getUserFromSession(String session) {
@ -207,13 +211,25 @@ public class DatabaseHelper {
});
Multithreading.runAsync(() -> {
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
}
CollectionUtils.collectPreloadedTabs(info.getTabs())
.stream()
.parallel()
.map(tab -> {
try {
return ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab).getRelatedItems();
} catch (ExtractionException | IOException e) {
throw new RuntimeException(e);
}
})
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.forEach(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
});
});
return channel;

5
src/main/java/me/kavin/piped/utils/obj/Playlist.java
View File

@ -4,14 +4,15 @@ import java.util.List;
public class Playlist {
public String name, thumbnailUrl, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar;
public String name, thumbnailUrl, description, bannerUrl, nextpage, uploader, uploaderUrl, uploaderAvatar;
public int videos;
public List<ContentItem> relatedStreams;
public Playlist(String name, String thumbnailUrl, String bannerUrl, String nextpage, String uploader,
public Playlist(String name, String thumbnailUrl, String description, String bannerUrl, String nextpage, String uploader,
String uploaderUrl, String uploaderAvatar, int videos, List<ContentItem> relatedStreams) {
this.name = name;
this.thumbnailUrl = thumbnailUrl;
this.description = description;
this.bannerUrl = bannerUrl;
this.nextpage = nextpage;
this.videos = videos;