TeamPiped-mirror/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2024-08-14 23:51:41 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #383 from TeamPiped/better-channel-updates

Browse source 
Query all videos at once and batch update
This commit is contained in:
Kavin 2022-10-06 11:09:32 +01:00 committed by GitHub
commit 12ccec3c4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 45 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View file

@ -74,6 +74,15 @@ public class DatabaseHelper {
return s.createQuery(cr).uniqueResult();
}
public static List<Video> getVideosFromIds(SharedSessionContract s, Collection<String> ids) {
CriteriaBuilder cb = s.getCriteriaBuilder();
CriteriaQuery<Video> cr = cb.createQuery(Video.class);
Root<Video> root = cr.from(Video.class);
cr.select(root).where(root.get("id").in(ids));
return s.createQuery(cr).list();
}
public static Video getVideoFromId(String id) {
try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
return getVideoFromId(s, id);

1
src/main/java/me/kavin/piped/utils/DownloaderImpl.java
View file

@ -30,6 +30,7 @@ public class DownloaderImpl extends Downloader {
final AsyncLoadingCache<Request, Response> responseCache = Caffeine.newBuilder()
.expireAfterWrite(1, TimeUnit.MINUTES)
.scheduler(Scheduler.systemScheduler())
.executor(Multithreading.getCachedExecutor())
.maximumSize(100).buildAsync(this::executeRequest);
@Override

38
src/main/java/me/kavin/piped/utils/ResponseHelper.java
View file

@ -37,6 +37,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
import org.schabi.newpipe.extractor.kiosk.KioskList;
@ -231,7 +232,7 @@ public class ResponseHelper {
me.kavin.piped.utils.obj.db.Channel channel = DatabaseHelper.getChannelFromId(info.getId());
try (Session s = DatabaseSessionFactory.createSession()) {
try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
if (channel != null) {
if (channel.isVerified() != info.isVerified()
@ -239,9 +240,29 @@ public class ResponseHelper {
channel.setVerified(info.isVerified());
channel.setUploaderAvatar(info.getAvatarUrl());
var tr = s.beginTransaction();
s.merge(channel);
s.update(channel);
tr.commit();
}
Set<String> ids = info.getRelatedItems()
.stream()
.filter(item -> {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
return System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION);
})
.map(item -> {
try {
return YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
} catch (ParsingException e) {
throw new RuntimeException(e);
}
})
.collect(Collectors.toUnmodifiableSet());
List<Video> videos = DatabaseHelper.getVideosFromIds(s, ids);
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
@ -249,7 +270,14 @@ public class ResponseHelper {
if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
try {
String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl());
updateVideo(id, item, time, true);
var video = videos.stream()
.filter(v -> v.getId().equals(id))
.findFirst();
if (video.isPresent()) {
updateVideo(s, video.get(), item);
} else {
handleNewVideo("https://youtube.com/watch?v=" + id, time, channel);
}
} catch (Exception e) {
ExceptionHandler.handle(e);
}
@ -1624,6 +1652,10 @@ public class ResponseHelper {
});
}
private static void updateVideo(StatelessSession s, Video video, StreamInfoItem item) {
updateVideo(s, video, item.getViewCount(), item.getDuration(), item.getName());
}
private static void updateVideo(StatelessSession s, Video video, long views, long duration, String title) {
boolean changed = false;