diff --git a/src/main/java/me/kavin/piped/ServerLauncher.java b/src/main/java/me/kavin/piped/ServerLauncher.java index 2df68e1..65d4554 100644 --- a/src/main/java/me/kavin/piped/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/ServerLauncher.java @@ -275,6 +275,14 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { } catch (Exception e) { return getErrorResponse(e, request.getPath()); } + })).map(POST, "/import/playlist", AsyncServlet.ofBlocking(executor, request -> { + try { + var json = Constants.mapper.readTree(request.loadBody().getResult().asArray()); + var playlistId = json.get("playlistId").textValue(); + return getJsonResponse(ResponseHelper.importPlaylistResponse(request.getHeader(AUTHORIZATION), playlistId), "private"); + } catch (Exception e) { + return getErrorResponse(e, request.getPath()); + } })).map(GET, "/subscriptions", AsyncServlet.ofBlocking(executor, request -> { try { return getJsonResponse(ResponseHelper.subscriptionsResponse(request.getHeader(AUTHORIZATION)), @@ -330,8 +338,8 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { } })).map(POST, "/user/delete", AsyncServlet.ofBlocking(executor, request -> { try { - DeleteUserRequest body = Constants.mapper.readValue(request.loadBody().getResult().asArray(), - DeleteUserRequest.class); + DeleteUserRequest body = Constants.mapper.readValue(request.loadBody().getResult().asArray(), + DeleteUserRequest.class); return getJsonResponse(ResponseHelper.deleteUserResponse(request.getHeader(AUTHORIZATION), body.password), "private"); } catch (Exception e) { diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 23f6611..a1a1a2e 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -82,7 +82,7 @@ public class Constants { hibernateProperties.put(key, value); }); - // transform hibernate properties for legacy configureations + // transform hibernate properties for legacy configurations hibernateProperties.replace("hibernate.dialect", "org.hibernate.dialect.PostgreSQL10Dialect", "org.hibernate.dialect.PostgreSQLDialect" diff --git a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java index 7564a05..e4e83aa 100644 --- a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java @@ -87,6 +87,15 @@ public class DatabaseHelper { return s.createQuery(cr).uniqueResult(); } + public static List getPlaylistVideosFromIds(Session s, List id) { + CriteriaBuilder cb = s.getCriteriaBuilder(); + CriteriaQuery cr = cb.createQuery(PlaylistVideo.class); + Root root = cr.from(PlaylistVideo.class); + cr.select(root).where(root.get("id").in(id)); + + return s.createQuery(cr).list(); + } + public static PubSub getPubSubFromId(Session s, String id) { CriteriaBuilder cb = s.getCriteriaBuilder(); CriteriaQuery cr = cb.createQuery(PubSub.class); diff --git a/src/main/java/me/kavin/piped/utils/ResponseHelper.java b/src/main/java/me/kavin/piped/utils/ResponseHelper.java index 8225bfe..1ba83b7 100644 --- a/src/main/java/me/kavin/piped/utils/ResponseHelper.java +++ b/src/main/java/me/kavin/piped/utils/ResponseHelper.java @@ -7,6 +7,7 @@ import com.grack.nanojson.JsonWriter; import com.rometools.rome.feed.synd.*; import com.rometools.rome.io.FeedException; import com.rometools.rome.io.SyndFeedOutput; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.CriteriaQuery; @@ -57,6 +58,7 @@ import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; +import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.utils.URLUtils.*; import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry; import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization; @@ -997,7 +999,7 @@ public class ResponseHelper { try (Session s = DatabaseSessionFactory.createSession()) { var playlist = new me.kavin.piped.utils.obj.db.Playlist(name, user, "https://i.ytimg.com/"); - s.save(playlist); + s.persist(playlist); s.getTransaction().begin(); s.getTransaction().commit(); @@ -1064,6 +1066,84 @@ public class ResponseHelper { } } + public static byte[] importPlaylistResponse(String session, String playlistId) throws IOException, ExtractionException { + + if (StringUtils.isBlank(playlistId)) + return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse()); + + var user = DatabaseHelper.getUserFromSession(session); + + if (user == null) + return Constants.mapper.writeValueAsBytes(new AuthenticationFailureResponse()); + + final String url = "https://www.youtube.com/playlist?list=" + playlistId; + + PlaylistInfo info = PlaylistInfo.getInfo(url); + + var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnailUrl()); + + List videos = new ObjectArrayList<>(info.getRelatedItems()); + + Page nextpage = info.getNextPage(); + + while (nextpage != null) { + var page = PlaylistInfo.getMoreItems(YOUTUBE_SERVICE, url, nextpage); + videos.addAll(page.getItems()); + + nextpage = page.getNextPage(); + } + + List channelIds = videos.stream() + .map(StreamInfoItem::getUploaderUrl) + .map(URLUtils::substringYouTube) + .map(s -> s.substring("/channel/".length())) + .collect(Collectors.toUnmodifiableSet()) + .stream() + .collect(Collectors.toUnmodifiableList()); + List videoIds = videos.stream() + .map(StreamInfoItem::getUrl) + .map(URLUtils::substringYouTube) + .map(s -> s.substring("/watch?v=".length())) + .collect(Collectors.toUnmodifiableList()); + + try (Session s = DatabaseSessionFactory.createSession()) { + + Map channelMap = new Object2ObjectOpenHashMap<>(); + + var channels = DatabaseHelper.getChannelsFromIds(s, channelIds); + channelIds.forEach(id -> { + var fetched = channels.stream().filter(channel -> channel.getUploaderId().equals(id)).findFirst() + .orElseGet(() -> saveChannel(id)); + channelMap.put(id, fetched); + }); + + Map videoMap = new Object2ObjectOpenHashMap<>(); + + var playlistVideos = DatabaseHelper.getPlaylistVideosFromIds(s, videoIds); + videoIds.forEach(id -> { + playlistVideos.stream().filter(video -> video.getId().equals(id)).findFirst() + .ifPresent(playlistVideo -> videoMap.put(id, playlistVideo)); + }); + + videos.forEach(video -> { + var channelId = substringYouTube(video.getUploaderUrl()).substring("/channel/".length()); + var videoId = substringYouTube(video.getUrl()).substring("/watch?v=".length()); + + var channel = channelMap.get(channelId); + + playlist.getVideos().add(videoMap.getOrDefault(videoId, new PlaylistVideo(videoId, video.getName(), video.getThumbnailUrl(), video.getDuration(), channel))); + }); + + var tr = s.beginTransaction(); + s.persist(playlist); + tr.commit(); + } + + return mapper.writeValueAsBytes(mapper.createObjectNode() + .put("playlistId", String.valueOf(playlist.getPlaylistId())) + ); + } + public static byte[] addToPlaylistResponse(String session, String playlistId, String videoId) throws IOException, ExtractionException { if (StringUtils.isBlank(playlistId) || StringUtils.isBlank(videoId)) @@ -1101,19 +1181,12 @@ public class ResponseHelper { var channel = DatabaseHelper.getChannelFromId(s, channelId); if (channel == null) { - ChannelInfo channelInfo = ChannelInfo.getInfo(info.getUploaderUrl()); - - channel = new me.kavin.piped.utils.obj.db.Channel(channelId, channelInfo.getName(), - channelInfo.getAvatarUrl(), channelInfo.isVerified()); - s.save(channel); - - if (!s.getTransaction().isActive()) - s.getTransaction().begin(); + channel = saveChannel(channelId); } video = new PlaylistVideo(videoId, info.getName(), info.getThumbnailUrl(), info.getDuration(), channel); - s.save(video); + s.persist(video); if (!s.getTransaction().isActive()) s.getTransaction().begin(); @@ -1296,31 +1369,38 @@ public class ResponseHelper { } } - private static void saveChannel(String channelId) { + private static me.kavin.piped.utils.obj.db.Channel saveChannel(String channelId) { try (Session s = DatabaseSessionFactory.createSession()) { - ChannelInfo info = null; + final ChannelInfo info; try { info = ChannelInfo.getInfo("https://youtube.com/channel/" + channelId); } catch (IOException | ExtractionException e) { ExceptionUtils.rethrow(e); + return null; } var channel = new me.kavin.piped.utils.obj.db.Channel(channelId, info.getName(), info.getAvatarUrl(), info.isVerified()); - s.save(channel); + s.persist(channel); s.beginTransaction().commit(); Multithreading.runAsync(() -> subscribePubSub(channelId)); - for (StreamInfoItem item : info.getRelatedItems()) { - long time = item.getUploadDate() != null - ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() - : System.currentTimeMillis(); - if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) - handleNewVideo(item.getUrl(), time, channel, s); - } + Multithreading.runAsync(() -> { + try (Session sess = DatabaseSessionFactory.createSession()) { + for (StreamInfoItem item : info.getRelatedItems()) { + long time = item.getUploadDate() != null + ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() + : System.currentTimeMillis(); + if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) + handleNewVideo(item.getUrl(), time, channel, sess); + } + } + }); + + return channel; } } diff --git a/src/main/java/me/kavin/piped/utils/obj/db/Playlist.java b/src/main/java/me/kavin/piped/utils/obj/db/Playlist.java index 129ac3d..c7b7c5a 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/Playlist.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/Playlist.java @@ -2,6 +2,7 @@ package me.kavin.piped.utils.obj.db; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import jakarta.persistence.*; +import org.hibernate.annotations.Cascade; import java.util.List; import java.util.UUID; @@ -46,6 +47,7 @@ public class Playlist { @Column(name = "videos") @CollectionTable(name = "playlists_videos_ids") @OrderColumn(name = "videos_order") + @Cascade(org.hibernate.annotations.CascadeType.PERSIST) private List videos; public long getId() { diff --git a/src/main/java/me/kavin/piped/utils/obj/db/PlaylistVideo.java b/src/main/java/me/kavin/piped/utils/obj/db/PlaylistVideo.java index c285d71..0a8c489 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/PlaylistVideo.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/PlaylistVideo.java @@ -28,7 +28,7 @@ public class PlaylistVideo { @Column(name = "duration") private long duration; - @Column(name = "thumbnail", length = 150) + @Column(name = "thumbnail", length = 400) private String thumbnail; @ManyToOne(fetch = FetchType.LAZY) diff --git a/src/main/java/me/kavin/piped/utils/obj/db/Video.java b/src/main/java/me/kavin/piped/utils/obj/db/Video.java index 49a87ce..6d12486 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/Video.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/Video.java @@ -24,7 +24,7 @@ public class Video { @Column(name = "uploaded") private long uploaded; - @Column(name = "thumbnail", length = 150) + @Column(name = "thumbnail", length = 400) private String thumbnail; @ManyToOne(fetch = FetchType.LAZY) diff --git a/testing/api-test.sh b/testing/api-test.sh index 27a17d7..6d92a19 100755 --- a/testing/api-test.sh +++ b/testing/api-test.sh @@ -108,5 +108,8 @@ curl ${CURLOPTS[@]} $HOST/user/playlists/remove -X POST -H "Content-Type: applic # Delete Playlist Test curl ${CURLOPTS[@]} $HOST/user/playlists/delete -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg playlistId $PLAYLIST_ID '{"playlistId": $playlistId}') || exit 1 +# Import Playlist Test +curl ${CURLOPTS[@]} $HOST/import/playlist -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg playlistId "PLQSoWXSpjA3-egtFq45DcUydZ885W7MTT" '{"playlistId": $playlistId}') || exit 1 + # Delete User Test curl ${CURLOPTS[@]} $HOST/user/delete -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg password "$PASS" '{"password": $password}') || exit 1