TeamPiped-mirror
/
Piped-Backend
mirror of https://github.com/TeamPiped/Piped-Backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes for the new new channel tab handler.

This commit is contained in:
Kavin 2023-05-22 20:59:25 +01:00
parent f6639511aa
commit 234adf1255
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
3 changed files with 92 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java
View File

@ -11,6 +11,7 @@ import me.kavin.piped.utils.obj.federation.FederatedVideoInfo;
import me.kavin.piped.utils.resp.InvalidRequestResponse;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
@ -28,6 +29,7 @@ import java.util.stream.Collectors;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs;
import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
import static me.kavin.piped.utils.URLUtils.rewriteURL;
@ -38,24 +40,30 @@ public class ChannelHandlers {
final ChannelInfo info = ChannelInfo.getInfo("https://youtube.com/" + channelPath);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getRelatedItems());
final ChannelTabInfo tabInfo = ChannelTabInfo.getInfo(YOUTUBE_SERVICE, collectPreloadedTabs(info.getTabs()).get(0));
Multithreading.runAsync(() -> info.getRelatedItems().forEach(infoItem -> {
if (
infoItem.getUploadDate() != null &&
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
< TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)
)
try {
MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo(
StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24),
infoItem.getName(),
infoItem.getDuration(), infoItem.getViewCount())
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}));
final List<ContentItem> relatedStreams = collectRelatedItems(tabInfo.getRelatedItems());
Multithreading.runAsync(() -> tabInfo.getRelatedItems()
.stream().filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.forEach(infoItem -> {
if (
infoItem.getUploadDate() != null &&
System.currentTimeMillis() - infoItem.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
< TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)
)
try {
MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo(
StringUtils.substring(infoItem.getUrl(), -11), StringUtils.substring(infoItem.getUploaderUrl(), -24),
infoItem.getName(),
infoItem.getDuration(), infoItem.getViewCount())
);
} catch (IOException e) {
throw new RuntimeException(e);
}
})
);
Multithreading.runAsync(() -> {
try {
@ -75,8 +83,10 @@ public class ChannelHandlers {
ChannelHelpers.updateChannel(s, channel, info.getName(), info.getAvatarUrl(), info.isVerified());
Set<String> ids = info.getRelatedItems()
Set<String> ids = tabInfo.getRelatedItems()
.stream()
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.filter(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
@ -94,32 +104,35 @@ public class ChannelHandlers {
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
try {
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
var video = videos.stream()
.filter(v -> v.getId().equals(id))
.findFirst();
if (video.isPresent()) {
VideoHelpers.updateVideo(id, item);
} else {
VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
}
tabInfo.getRelatedItems()
.stream()
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast).forEach(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
try {
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
var video = videos.stream()
.filter(v -> v.getId().equals(id))
.findFirst();
if (video.isPresent()) {
VideoHelpers.updateVideo(id, item);
} else {
VideoHelpers.handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
});
}
}
});
String nextpage = null;
if (info.hasNextPage()) {
Page page = info.getNextPage();
if (tabInfo.hasNextPage()) {
Page page = tabInfo.getNextPage();
nextpage = mapper.writeValueAsString(page);
}
@ -152,8 +165,11 @@ public class ChannelHandlers {
if (prevpage == null)
ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter"));
ListExtractor.InfoItemsPage<StreamInfoItem> info = ChannelInfo.getMoreItems(YOUTUBE_SERVICE,
"https://youtube.com/channel/" + channelId, prevpage);
String channelUrl = "https://youtube.com/channel/" + channelId;
String url = channelUrl + "/videos";
ListExtractor.InfoItemsPage<InfoItem> info = ChannelTabInfo.getMoreItems(YOUTUBE_SERVICE,
new ListLinkHandler(url, url, channelId, List.of("videos"), ""), prevpage);
final List<ContentItem> relatedStreams = collectRelatedItems(info.getItems());

12
src/main/java/me/kavin/piped/utils/CollectionUtils.java
View File

@ -4,6 +4,9 @@ import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.kavin.piped.utils.obj.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -13,6 +16,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Optional;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.utils.URLUtils.*;
public class CollectionUtils {
@ -122,4 +126,12 @@ public class CollectionUtils {
item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
item.isVerified());
}
public static List<ReadyChannelTabListLinkHandler> collectPreloadedTabs(List<ListLinkHandler> tabs) {
return tabs
.stream()
.filter(ReadyChannelTabListLinkHandler.class::isInstance)
.map(ReadyChannelTabListLinkHandler.class::cast)
.toList();
}
}

30
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View File

@ -9,7 +9,9 @@ import me.kavin.piped.utils.obj.db.*;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hibernate.SharedSessionContract;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.ChannelTabInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@ -19,6 +21,8 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
public class DatabaseHelper {
public static User getUserFromSession(String session) {
@ -207,13 +211,25 @@ public class DatabaseHelper {
});
Multithreading.runAsync(() -> {
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
}
CollectionUtils.collectPreloadedTabs(info.getTabs())
.stream()
.parallel()
.map(tab -> {
try {
return ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab).getRelatedItems();
} catch (ExtractionException | IOException e) {
throw new RuntimeException(e);
}
})
.filter(StreamInfoItem.class::isInstance)
.map(StreamInfoItem.class::cast)
.forEach(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
VideoHelpers.handleNewVideo(item.getUrl(), time, channel);
});
});
return channel;