Merge pull request #719 from TeamPiped/npe-update

Update NPE

build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation 'it.unimi.dsi:fastutil-core:8.5.12'
     implementation 'commons-codec:commons-codec:1.16.0'
     implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
-    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:5518112dce594bb4cd07d8ed3391bcc5e688f829'
+    implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:48beff184a9792c4787cfa05fce577c3adf89f56'
     implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
     implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
     implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'

src/main/java/me/kavin/piped/Main.java

@@ -18,7 +18,7 @@ import org.hibernate.StatelessSession;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 
 import java.util.*;
@@ -62,8 +62,8 @@ public class Main {
         new Timer().scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
-                System.out.printf("ThrottlingCache: %o entries%n", YoutubeThrottlingDecrypter.getCacheSize());
+                System.out.printf("ThrottlingCache: %o entries%n", YoutubeJavaScriptPlayerManager.getThrottlingParametersCacheSize());
-                YoutubeThrottlingDecrypter.clearCache();
+                YoutubeJavaScriptPlayerManager.clearThrottlingParametersCache();
             }
         }, 0, TimeUnit.MINUTES.toMillis(60));
 

src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java

@@ -32,7 +32,7 @@ import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
 import static me.kavin.piped.consts.Constants.mapper;
 import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs;
 import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
-import static me.kavin.piped.utils.URLUtils.rewriteURL;
+import static me.kavin.piped.utils.URLUtils.getLastThumbnail;
 
 public class ChannelHandlers {
     public static byte[] channelResponse(String channelPath) throws Exception {
@@ -77,7 +77,7 @@ public class ChannelHandlers {
         Multithreading.runAsync(() -> {
             try {
                 MatrixHelper.sendEvent("video.piped.channel.info", new FederatedChannelInfo(
-                        info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified())
+                        info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified())
                 );
             } catch (IOException e) {
                 throw new RuntimeException(e);
@@ -93,7 +93,7 @@ public class ChannelHandlers {
 
                     if (channel != null) {
 
-                        ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified());
+                        ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified());
 
                         Set<String> ids = tabInfo.getRelatedItems()
                                 .stream()
@@ -159,8 +159,8 @@ public class ChannelHandlers {
                     }
                 }).toList();
 
-        final Channel channel = new Channel(info.getId(), info.getName(), rewriteURL(info.getAvatarUrl()),
+        final Channel channel = new Channel(info.getId(), info.getName(), getLastThumbnail(info.getAvatars()),
-                rewriteURL(info.getBannerUrl()), info.getDescription(), info.getSubscriberCount(), info.isVerified(),
+                getLastThumbnail(info.getBanners()), info.getDescription(), info.getSubscriberCount(), info.isVerified(),
                 nextpage, relatedStreams, tabs);
 
         return mapper.writeValueAsBytes(channel);

src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java

@@ -30,8 +30,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
 import static me.kavin.piped.consts.Constants.mapper;
 import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems;
-import static me.kavin.piped.utils.URLUtils.rewriteURL;
+import static me.kavin.piped.utils.URLUtils.*;
-import static me.kavin.piped.utils.URLUtils.substringYouTube;
 
 public class PlaylistHandlers {
     public static byte[] playlistResponse(String playlistId) throws Exception {
@@ -60,10 +59,10 @@ public class PlaylistHandlers {
             nextpage = mapper.writeValueAsString(page);
         }
 
-        final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()),
+        final Playlist playlist = new Playlist(info.getName(), getLastThumbnail(info.getThumbnails()),
-                info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage,
+                info.getDescription().getContent(), getLastThumbnail(info.getBanners()), nextpage,
                 info.getUploaderName().isEmpty() ? null : info.getUploaderName(),
-                substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()),
+                substringYouTube(info.getUploaderUrl()), getLastThumbnail(info.getUploaderAvatars()),
                 (int) info.getStreamCount(), relatedStreams);
 
         return mapper.writeValueAsBytes(playlist);

src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java

@@ -36,8 +36,7 @@ import java.util.concurrent.TimeoutException;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
 import static me.kavin.piped.consts.Constants.mapper;
-import static me.kavin.piped.utils.URLUtils.rewriteURL;
+import static me.kavin.piped.utils.URLUtils.*;
-import static me.kavin.piped.utils.URLUtils.substringYouTube;
 import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry;
 import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
@@ -342,7 +341,7 @@ public class StreamHandlers {
                 if (comment.getReplies() != null)
                     repliespage = mapper.writeValueAsString(comment.getReplies());
 
-                comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()),
+                comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()),
                         comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(),
                         substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(),
                         comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified()));
@@ -380,7 +379,7 @@ public class StreamHandlers {
                 if (comment.getReplies() != null)
                     repliespage = mapper.writeValueAsString(comment.getReplies());
 
-                comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()),
+                comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()),
                         comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(),
                         substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(),
                         comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified()));

src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java

@@ -268,7 +268,7 @@ public class AuthPlaylistHandlers {
                             channel = DatabaseHelper.saveChannel(channelId);
                         }
 
-                        video = new PlaylistVideo(videoId, info.getName(), info.getThumbnailUrl(), info.getDuration(), channel);
+                        video = new PlaylistVideo(videoId, info.getName(), info.getThumbnails().getLast().getUrl(), info.getDuration(), channel);
 
                         var tr = s.beginTransaction();
                         try {
@@ -402,7 +402,7 @@ public class AuthPlaylistHandlers {
 
         PlaylistInfo info = PlaylistInfo.getInfo(url);
 
-        var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnailUrl());
+        var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnails().getLast().getUrl());
 
         List<StreamInfoItem> videos = new ObjectArrayList<>(info.getRelatedItems());
 
@@ -451,7 +451,7 @@ public class AuthPlaylistHandlers {
 
                 var channel = channelMap.get(channelId);
 
-                playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnailUrl(), video.getDuration(), channel)));
+                playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnails().getLast().getUrl(), video.getDuration(), channel)));
             });
 
             var tr = s.beginTransaction();

src/main/java/me/kavin/piped/utils/CollectionUtils.java

@@ -71,7 +71,7 @@ public class CollectionUtils {
 
         return new Streams(info.getName(), info.getDescription().getContent(),
                 info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()),
-                rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(),
+                getLastThumbnail(info.getUploaderAvatars()), getLastThumbnail(info.getThumbnails()), info.getDuration(),
                 info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(),
                 audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()),
                 rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(),
@@ -101,9 +101,9 @@ public class CollectionUtils {
         StreamInfoItem item = (StreamInfoItem) o;
 
         return new StreamItem(substringYouTube(item.getUrl()), item.getName(),
-                rewriteURL(item.getThumbnailUrl()),
+                getLastThumbnail(item.getThumbnails()),
                 item.getUploaderName(), substringYouTube(item.getUploaderUrl()),
-                rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(),
+                getLastThumbnail(item.getUploaderAvatars()), item.getTextualUploadDate(),
                 item.getShortDescription(), item.getDuration(),
                 item.getViewCount(), item.getUploadDate() != null ?
                 item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : -1,
@@ -115,7 +115,7 @@ public class CollectionUtils {
         PlaylistInfoItem item = (PlaylistInfoItem) o;
 
         return new PlaylistItem(substringYouTube(item.getUrl()), item.getName(),
-                rewriteURL(item.getThumbnailUrl()),
+                getLastThumbnail(item.getThumbnails()),
                 item.getUploaderName(), substringYouTube(item.getUploaderUrl()),
                 item.isUploaderVerified(),
                 item.getPlaylistType().name(), item.getStreamCount());
@@ -126,7 +126,7 @@ public class CollectionUtils {
         ChannelInfoItem item = (ChannelInfoItem) o;
 
         return new ChannelItem(substringYouTube(item.getUrl()), item.getName(),
-                rewriteURL(item.getThumbnailUrl()),
+                getLastThumbnail(item.getThumbnails()),
                 item.getDescription(), item.getSubscriberCount(), item.getStreamCount(),
                 item.isVerified());
     }

src/main/java/me/kavin/piped/utils/DatabaseHelper.java

@@ -192,7 +192,7 @@ public class DatabaseHelper {
         }
 
         var channel = new Channel(channelId, StringUtils.abbreviate(info.getName(), 100),
-                info.getAvatarUrl(), info.isVerified());
+                info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified());
 
         try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {
             var tr = s.beginTransaction();

src/main/java/me/kavin/piped/utils/URLUtils.java

@@ -2,12 +2,14 @@ package me.kavin.piped.utils;
 
 import me.kavin.piped.consts.Constants;
 import org.apache.commons.lang3.StringUtils;
+import org.schabi.newpipe.extractor.Image;
 
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 public class URLUtils {
 
@@ -37,6 +39,10 @@ public class URLUtils {
         return rewriteURL(old, Constants.IMAGE_PROXY_PART);
     }
 
+    public static String getLastThumbnail(final List<Image> thumbnails) {
+        return thumbnails.isEmpty() ? null : rewriteURL(thumbnails.getLast().getUrl());
+    }
+
     public static String rewriteVideoURL(final String old) {
         return rewriteURL(old, Constants.PROXY_PART);
     }

src/main/java/me/kavin/piped/utils/VideoHelpers.java

@@ -50,7 +50,7 @@ public class VideoHelpers {
                 if (!DatabaseHelper.doesVideoExist(s, info.getId())) {
 
                     Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(),
-                            Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel);
+                            Math.max(infoTime, time), info.getThumbnails().getLast().getUrl(), info.isShortFormContent(), channel);
 
                     insertVideo(video);
                     return;
@@ -81,7 +81,7 @@ public class VideoHelpers {
                     boolean isShort = extractor.isShortFormContent() || isShort(extractor.getId());
 
                     Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(),
-                            Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel);
+                            Math.max(infoTime, time), extractor.getThumbnails().getLast().getUrl(), isShort, channel);
 
                     insertVideo(video);