TeamPiped-mirror
/
Piped-Backend
mirror of https://github.com/TeamPiped/Piped-Backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add route to import playlist from YouTube. (#282)

This commit is contained in:
Kavin 2022-06-18 19:10:26 +01:00 committed by GitHub
parent 457a68fa40
commit 18211a3eed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 127 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/me/kavin/piped/ServerLauncher.java
View File

@ -275,6 +275,14 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
} catch (Exception e) {
return getErrorResponse(e, request.getPath());
}
})).map(POST, "/import/playlist", AsyncServlet.ofBlocking(executor, request -> {
try {
var json = Constants.mapper.readTree(request.loadBody().getResult().asArray());
var playlistId = json.get("playlistId").textValue();
return getJsonResponse(ResponseHelper.importPlaylistResponse(request.getHeader(AUTHORIZATION), playlistId), "private");
} catch (Exception e) {
return getErrorResponse(e, request.getPath());
}
})).map(GET, "/subscriptions", AsyncServlet.ofBlocking(executor, request -> {
try {
return getJsonResponse(ResponseHelper.subscriptionsResponse(request.getHeader(AUTHORIZATION)),
@ -330,8 +338,8 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
}
})).map(POST, "/user/delete", AsyncServlet.ofBlocking(executor, request -> {
try {
DeleteUserRequest body = Constants.mapper.readValue(request.loadBody().getResult().asArray(),
DeleteUserRequest.class);
DeleteUserRequest body = Constants.mapper.readValue(request.loadBody().getResult().asArray(),
DeleteUserRequest.class);
return getJsonResponse(ResponseHelper.deleteUserResponse(request.getHeader(AUTHORIZATION), body.password),
"private");
} catch (Exception e) {

2
src/main/java/me/kavin/piped/consts/Constants.java
View File

@ -82,7 +82,7 @@ public class Constants {
hibernateProperties.put(key, value);
});
// transform hibernate properties for legacy configureations
// transform hibernate properties for legacy configurations
hibernateProperties.replace("hibernate.dialect",
"org.hibernate.dialect.PostgreSQL10Dialect",
"org.hibernate.dialect.PostgreSQLDialect"

9
src/main/java/me/kavin/piped/utils/DatabaseHelper.java
View File

@ -87,6 +87,15 @@ public class DatabaseHelper {
return s.createQuery(cr).uniqueResult();
}
public static List<PlaylistVideo> getPlaylistVideosFromIds(Session s, List<String> id) {
CriteriaBuilder cb = s.getCriteriaBuilder();
CriteriaQuery<PlaylistVideo> cr = cb.createQuery(PlaylistVideo.class);
Root<PlaylistVideo> root = cr.from(PlaylistVideo.class);
cr.select(root).where(root.get("id").in(id));
return s.createQuery(cr).list();
}
public static PubSub getPubSubFromId(Session s, String id) {
CriteriaBuilder cb = s.getCriteriaBuilder();
CriteriaQuery<PubSub> cr = cb.createQuery(PubSub.class);

120
src/main/java/me/kavin/piped/utils/ResponseHelper.java
View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonWriter;
import com.rometools.rome.feed.synd.*;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedOutput;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaQuery;
@ -57,6 +58,7 @@ import java.util.stream.Collectors;
import static java.nio.charset.StandardCharsets.UTF_8;
import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.URLUtils.*;
import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry;
import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization;
@ -997,7 +999,7 @@ public class ResponseHelper {
try (Session s = DatabaseSessionFactory.createSession()) {
var playlist = new me.kavin.piped.utils.obj.db.Playlist(name, user, "https://i.ytimg.com/");
s.save(playlist);
s.persist(playlist);
s.getTransaction().begin();
s.getTransaction().commit();
@ -1064,6 +1066,84 @@ public class ResponseHelper {
}
}
public static byte[] importPlaylistResponse(String session, String playlistId) throws IOException, ExtractionException {
if (StringUtils.isBlank(playlistId))
return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse());
var user = DatabaseHelper.getUserFromSession(session);
if (user == null)
return Constants.mapper.writeValueAsBytes(new AuthenticationFailureResponse());
final String url = "https://www.youtube.com/playlist?list=" + playlistId;
PlaylistInfo info = PlaylistInfo.getInfo(url);
var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnailUrl());
List<StreamInfoItem> videos = new ObjectArrayList<>(info.getRelatedItems());
Page nextpage = info.getNextPage();
while (nextpage != null) {
var page = PlaylistInfo.getMoreItems(YOUTUBE_SERVICE, url, nextpage);
videos.addAll(page.getItems());
nextpage = page.getNextPage();
}
List<String> channelIds = videos.stream()
.map(StreamInfoItem::getUploaderUrl)
.map(URLUtils::substringYouTube)
.map(s -> s.substring("/channel/".length()))
.collect(Collectors.toUnmodifiableSet())
.stream()
.collect(Collectors.toUnmodifiableList());
List<String> videoIds = videos.stream()
.map(StreamInfoItem::getUrl)
.map(URLUtils::substringYouTube)
.map(s -> s.substring("/watch?v=".length()))
.collect(Collectors.toUnmodifiableList());
try (Session s = DatabaseSessionFactory.createSession()) {
Map<String, me.kavin.piped.utils.obj.db.Channel> channelMap = new Object2ObjectOpenHashMap<>();
var channels = DatabaseHelper.getChannelsFromIds(s, channelIds);
channelIds.forEach(id -> {
var fetched = channels.stream().filter(channel -> channel.getUploaderId().equals(id)).findFirst()
.orElseGet(() -> saveChannel(id));
channelMap.put(id, fetched);
});
Map<String, PlaylistVideo> videoMap = new Object2ObjectOpenHashMap<>();
var playlistVideos = DatabaseHelper.getPlaylistVideosFromIds(s, videoIds);
videoIds.forEach(id -> {
playlistVideos.stream().filter(video -> video.getId().equals(id)).findFirst()
.ifPresent(playlistVideo -> videoMap.put(id, playlistVideo));
});
videos.forEach(video -> {
var channelId = substringYouTube(video.getUploaderUrl()).substring("/channel/".length());
var videoId = substringYouTube(video.getUrl()).substring("/watch?v=".length());
var channel = channelMap.get(channelId);
playlist.getVideos().add(videoMap.getOrDefault(videoId, new PlaylistVideo(videoId, video.getName(), video.getThumbnailUrl(), video.getDuration(), channel)));
});
var tr = s.beginTransaction();
s.persist(playlist);
tr.commit();
}
return mapper.writeValueAsBytes(mapper.createObjectNode()
.put("playlistId", String.valueOf(playlist.getPlaylistId()))
);
}
public static byte[] addToPlaylistResponse(String session, String playlistId, String videoId) throws IOException, ExtractionException {
if (StringUtils.isBlank(playlistId) || StringUtils.isBlank(videoId))
@ -1101,19 +1181,12 @@ public class ResponseHelper {
var channel = DatabaseHelper.getChannelFromId(s, channelId);
if (channel == null) {
ChannelInfo channelInfo = ChannelInfo.getInfo(info.getUploaderUrl());
channel = new me.kavin.piped.utils.obj.db.Channel(channelId, channelInfo.getName(),
channelInfo.getAvatarUrl(), channelInfo.isVerified());
s.save(channel);
if (!s.getTransaction().isActive())
s.getTransaction().begin();
channel = saveChannel(channelId);
}
video = new PlaylistVideo(videoId, info.getName(), info.getThumbnailUrl(), info.getDuration(), channel);
s.save(video);
s.persist(video);
if (!s.getTransaction().isActive())
s.getTransaction().begin();
@ -1296,31 +1369,38 @@ public class ResponseHelper {
}
}
private static void saveChannel(String channelId) {
private static me.kavin.piped.utils.obj.db.Channel saveChannel(String channelId) {
try (Session s = DatabaseSessionFactory.createSession()) {
ChannelInfo info = null;
final ChannelInfo info;
try {
info = ChannelInfo.getInfo("https://youtube.com/channel/" + channelId);
} catch (IOException | ExtractionException e) {
ExceptionUtils.rethrow(e);
return null;
}
var channel = new me.kavin.piped.utils.obj.db.Channel(channelId, info.getName(),
info.getAvatarUrl(), info.isVerified());
s.save(channel);
s.persist(channel);
s.beginTransaction().commit();
Multithreading.runAsync(() -> subscribePubSub(channelId));
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
handleNewVideo(item.getUrl(), time, channel, s);
}
Multithreading.runAsync(() -> {
try (Session sess = DatabaseSessionFactory.createSession()) {
for (StreamInfoItem item : info.getRelatedItems()) {
long time = item.getUploadDate() != null
? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
: System.currentTimeMillis();
if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
handleNewVideo(item.getUrl(), time, channel, sess);
}
}
});
return channel;
}
}

2
src/main/java/me/kavin/piped/utils/obj/db/Playlist.java
View File

@ -2,6 +2,7 @@ package me.kavin.piped.utils.obj.db;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import jakarta.persistence.*;
import org.hibernate.annotations.Cascade;
import java.util.List;
import java.util.UUID;
@ -46,6 +47,7 @@ public class Playlist {
@Column(name = "videos")
@CollectionTable(name = "playlists_videos_ids")
@OrderColumn(name = "videos_order")
@Cascade(org.hibernate.annotations.CascadeType.PERSIST)
private List<PlaylistVideo> videos;
public long getId() {

2
src/main/java/me/kavin/piped/utils/obj/db/PlaylistVideo.java
View File

@ -28,7 +28,7 @@ public class PlaylistVideo {
@Column(name = "duration")
private long duration;
@Column(name = "thumbnail", length = 150)
@Column(name = "thumbnail", length = 400)
private String thumbnail;
@ManyToOne(fetch = FetchType.LAZY)

2
src/main/java/me/kavin/piped/utils/obj/db/Video.java
View File

@ -24,7 +24,7 @@ public class Video {
@Column(name = "uploaded")
private long uploaded;
@Column(name = "thumbnail", length = 150)
@Column(name = "thumbnail", length = 400)
private String thumbnail;
@ManyToOne(fetch = FetchType.LAZY)

3
testing/api-test.sh
View File

@ -108,5 +108,8 @@ curl ${CURLOPTS[@]} $HOST/user/playlists/remove -X POST -H "Content-Type: applic
# Delete Playlist Test
curl ${CURLOPTS[@]} $HOST/user/playlists/delete -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg playlistId $PLAYLIST_ID '{"playlistId": $playlistId}') || exit 1
# Import Playlist Test
curl ${CURLOPTS[@]} $HOST/import/playlist -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg playlistId "PLQSoWXSpjA3-egtFq45DcUydZ885W7MTT" '{"playlistId": $playlistId}') || exit 1
# Delete User Test
curl ${CURLOPTS[@]} $HOST/user/delete -X POST -H "Content-Type: application/json" -H "Authorization: $AUTH_TOKEN" -d $(jq -n --compact-output --arg password "$PASS" '{"password": $password}') || exit 1