From 71403e93af96ab9c86826a4cd01fd24c9e21d410 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 13 Aug 2023 23:20:30 +0100 Subject: [PATCH 01/24] Add liquibase to perform database migrations --- build.gradle | 1 + src/main/java/me/kavin/piped/Main.java | 8 ++- .../me/kavin/piped/utils/LiquibaseHelper.java | 51 +++++++++++++++++++ .../changelog/db.changelog-master.xml | 9 ++++ .../changelog/version/0-1-init-crdb.sql | 3 ++ .../changelog/version/0-1-init-pg.sql | 3 ++ .../resources/changelog/version/0-1-init.sql | 11 ++++ .../resources/changelog/version/0-init.xml | 13 +++++ src/main/resources/hibernate.cfg.xml | 4 +- 9 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 src/main/java/me/kavin/piped/utils/LiquibaseHelper.java create mode 100644 src/main/resources/changelog/db.changelog-master.xml create mode 100644 src/main/resources/changelog/version/0-1-init-crdb.sql create mode 100644 src/main/resources/changelog/version/0-1-init-pg.sql create mode 100644 src/main/resources/changelog/version/0-1-init.sql create mode 100644 src/main/resources/changelog/version/0-init.xml diff --git a/build.gradle b/build.gradle index cb9a478..332d724 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation 'org.postgresql:postgresql:42.6.0' implementation 'org.hibernate:hibernate-core:6.2.7.Final' implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' + implementation 'org.liquibase:liquibase-core:4.23.1' implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.2' implementation 'commons-logging:commons-logging:1.2' diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index 067f229..15e982c 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -20,7 +20,6 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import rocks.kavin.reqwest4j.ReqwestUtils; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -70,6 +69,13 @@ public class Main { } }).start(); + try { + LiquibaseHelper.init(); + } catch (Exception e) { + ExceptionHandler.handle(e); + System.exit(1); + } + try (Session ignored = DatabaseSessionFactory.createSession()) { System.out.println("Database connection is ready!"); } catch (Throwable t) { diff --git a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java new file mode 100644 index 0000000..e585799 --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java @@ -0,0 +1,51 @@ +package me.kavin.piped.utils; + +import liquibase.Contexts; +import liquibase.Liquibase; +import liquibase.Scope; +import liquibase.command.CommandScope; +import liquibase.command.core.UpdateCommandStep; +import liquibase.command.core.helpers.ChangeExecListenerCommandStep; +import liquibase.command.core.helpers.DatabaseChangelogCommandStep; +import liquibase.command.core.helpers.DbUrlConnectionCommandStep; +import liquibase.database.Database; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.exception.LiquibaseException; +import liquibase.resource.ClassLoaderResourceAccessor; +import me.kavin.piped.consts.Constants; + +import java.io.FileWriter; +import java.io.IOException; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +public class LiquibaseHelper { + + public static void init() throws Exception { + + String url = Constants.hibernateProperties.get("hibernate.connection.url"); + String username = Constants.hibernateProperties.get("hibernate.connection.username"); + String password = Constants.hibernateProperties.get("hibernate.connection.password"); + + Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(DriverManager.getConnection(url, username, password))); + + try (Liquibase liquibase = new Liquibase("changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), database)) { + + Map scopeObjects = new HashMap<>(); + scopeObjects.put(Scope.Attr.database.name(), liquibase.getDatabase()); + scopeObjects.put(Scope.Attr.resourceAccessor.name(), liquibase.getResourceAccessor()); + + Scope.child(scopeObjects, () -> { + CommandScope updateCommand = new CommandScope(UpdateCommandStep.COMMAND_NAME); + updateCommand.addArgumentValue(DbUrlConnectionCommandStep.DATABASE_ARG, liquibase.getDatabase()); + updateCommand.addArgumentValue(UpdateCommandStep.CHANGELOG_FILE_ARG, liquibase.getChangeLogFile()); + updateCommand.execute(); + }); + + } + } + +} diff --git a/src/main/resources/changelog/db.changelog-master.xml b/src/main/resources/changelog/db.changelog-master.xml new file mode 100644 index 0000000..4478527 --- /dev/null +++ b/src/main/resources/changelog/db.changelog-master.xml @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql new file mode 100644 index 0000000..ec15eae --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC) STORING (password, username); + +--rollback DROP INDEX IF EXISTS users_session_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql new file mode 100644 index 0000000..610e4c2 --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC); + +--rollback DROP INDEX IF EXISTS users_session_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql new file mode 100644 index 0000000..d01cfee --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS users ( + id SERIAL NOT NULL, + password STRING NULL, + session_id STRING(36) NULL, + username STRING(24) NULL UNIQUE, + CONSTRAINT users_pkey PRIMARY KEY (id ASC), + INDEX users_id_idx (id ASC), + INDEX username_idx (username ASC) +); + +--rollback DROP TABLE IF EXISTS users; diff --git a/src/main/resources/changelog/version/0-init.xml b/src/main/resources/changelog/version/0-init.xml new file mode 100644 index 0000000..fc0fe76 --- /dev/null +++ b/src/main/resources/changelog/version/0-init.xml @@ -0,0 +1,13 @@ + + + + + + + + + + diff --git a/src/main/resources/hibernate.cfg.xml b/src/main/resources/hibernate.cfg.xml index a0953d0..6374737 100644 --- a/src/main/resources/hibernate.cfg.xml +++ b/src/main/resources/hibernate.cfg.xml @@ -4,11 +4,11 @@ "http://www.hibernate.org/dtd/hibernate-configuration-3.0.dtd"> - update + validate false true - + org.hibernate.hikaricp.internal.HikariCPConnectionProvider DELAYED_ACQUISITION_AND_RELEASE_AFTER_STATEMENT 50 From d81bb969c7e069bb81746e34463c4acc77e929c1 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 13 Aug 2023 23:31:42 +0100 Subject: [PATCH 02/24] Register postgres driver. --- .../java/me/kavin/piped/utils/LiquibaseHelper.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java index e585799..f412ff8 100644 --- a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java @@ -1,24 +1,17 @@ package me.kavin.piped.utils; -import liquibase.Contexts; import liquibase.Liquibase; import liquibase.Scope; import liquibase.command.CommandScope; import liquibase.command.core.UpdateCommandStep; -import liquibase.command.core.helpers.ChangeExecListenerCommandStep; -import liquibase.command.core.helpers.DatabaseChangelogCommandStep; import liquibase.command.core.helpers.DbUrlConnectionCommandStep; import liquibase.database.Database; import liquibase.database.DatabaseFactory; import liquibase.database.jvm.JdbcConnection; -import liquibase.exception.LiquibaseException; import liquibase.resource.ClassLoaderResourceAccessor; import me.kavin.piped.consts.Constants; -import java.io.FileWriter; -import java.io.IOException; import java.sql.DriverManager; -import java.sql.SQLException; import java.util.HashMap; import java.util.Map; @@ -30,6 +23,9 @@ public class LiquibaseHelper { String username = Constants.hibernateProperties.get("hibernate.connection.username"); String password = Constants.hibernateProperties.get("hibernate.connection.password"); + // ensure postgres driver is loaded + DriverManager.registerDriver(new org.postgresql.Driver()); + Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(DriverManager.getConnection(url, username, password))); try (Liquibase liquibase = new Liquibase("changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), database)) { From 8f6a954ccac9d8d3133c377d7e882485c8a1cf14 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sun, 13 Aug 2023 23:54:42 +0100 Subject: [PATCH 03/24] Fix datatypes and drop unused duplicate index. --- .../java/me/kavin/piped/utils/obj/db/User.java | 5 +++-- src/main/resources/changelog/version/0-1-init.sql | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/me/kavin/piped/utils/obj/db/User.java b/src/main/java/me/kavin/piped/utils/obj/db/User.java index fe3ceb4..2b8f603 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/User.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/User.java @@ -8,9 +8,10 @@ import java.util.Set; import java.util.UUID; @Entity -@Table(name = "users", indexes = {@Index(columnList = "id", name = "users_id_idx"), +@Table(name = "users", indexes = { @Index(columnList = "username", name = "username_idx"), - @Index(columnList = "session_id", name = "users_session_id_idx")}) + @Index(columnList = "session_id", name = "users_session_id_idx") +}) public class User implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index d01cfee..e44fd80 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -1,11 +1,13 @@ CREATE TABLE IF NOT EXISTS users ( id SERIAL NOT NULL, - password STRING NULL, - session_id STRING(36) NULL, - username STRING(24) NULL UNIQUE, - CONSTRAINT users_pkey PRIMARY KEY (id ASC), - INDEX users_id_idx (id ASC), - INDEX username_idx (username ASC) + password TEXT NULL, + session_id VARCHAR(36) NULL, + username VARCHAR(24) NULL UNIQUE, + CONSTRAINT users_pkey PRIMARY KEY (id) ); +DROP INDEX IF EXISTS users_id_idx; + +CREATE INDEX IF NOT EXISTS username_idx ON users (username ASC); + --rollback DROP TABLE IF EXISTS users; From 30d7c0323d86ea26d75bc19f5754c62ee0a30f13 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 06:29:18 +0100 Subject: [PATCH 04/24] Add yugabytedb support. --- build.gradle | 1 + src/main/resources/changelog/version/0-init.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 332d724..b55eeb3 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ dependencies { implementation 'org.hibernate:hibernate-core:6.2.7.Final' implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' implementation 'org.liquibase:liquibase-core:4.23.1' + implementation 'org.liquibase.ext:liquibase-yugabytedb:4.23.0' implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.2' implementation 'commons-logging:commons-logging:1.2' diff --git a/src/main/resources/changelog/version/0-init.xml b/src/main/resources/changelog/version/0-init.xml index fc0fe76..68b90cc 100644 --- a/src/main/resources/changelog/version/0-init.xml +++ b/src/main/resources/changelog/version/0-init.xml @@ -7,7 +7,7 @@ - + From 3f4d431f7b3c1801dfd05e31d436a50f656afd6f Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 07:00:29 +0100 Subject: [PATCH 05/24] Set max time to 60 seconds. --- testing/api-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/api-test.sh b/testing/api-test.sh index df3b9be..1f04525 100755 --- a/testing/api-test.sh +++ b/testing/api-test.sh @@ -1,6 +1,6 @@ #!/bin/bash -CURLOPTS=(-i -s -S -o /dev/null -f -w "%{http_code}\tTime:\t%{time_starttransfer}\t%{url_effective}\n") +CURLOPTS=(-i -s -S --max-time 60 -o /dev/null -f -w "%{http_code}\tTime:\t%{time_starttransfer}\t%{url_effective}\n") HOST="127.0.0.1:8080" # Healthcheck Test From e02644a45ba35a862ee37ff990d4cb50cffb3b1b Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 07:28:29 +0100 Subject: [PATCH 06/24] Exclude liquibase dependency in extension. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b55eeb3..2d8dae0 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ dependencies { implementation 'org.hibernate:hibernate-core:6.2.7.Final' implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' implementation 'org.liquibase:liquibase-core:4.23.1' - implementation 'org.liquibase.ext:liquibase-yugabytedb:4.23.0' + implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.0') { exclude group: 'org.liquibase' } implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.2' implementation 'commons-logging:commons-logging:1.2' From 8258468f7b8a58b64924f103440c32dc06440943 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:20:40 +0100 Subject: [PATCH 07/24] Implement channels table. --- src/main/resources/changelog/version/0-1-init.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index e44fd80..6e6eb01 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -10,4 +10,18 @@ DROP INDEX IF EXISTS users_id_idx; CREATE INDEX IF NOT EXISTS username_idx ON users (username ASC); +--rollback DROP INDEX IF EXISTS username_idx; --rollback DROP TABLE IF EXISTS users; + +CREATE TABLE IF NOT EXISTS channels ( + uploader_id VARCHAR(24) NOT NULL, + uploader VARCHAR(100) NULL, + uploader_avatar VARCHAR(150) NULL, + verified BOOL NULL, + CONSTRAINT channels_pkey PRIMARY KEY (uploader_id) +); + +CREATE INDEX IF NOT EXISTS channels_uploader_idx ON channels (uploader ASC); + +--rollback DROP INDEX IF EXISTS channels_uploader_idx; +--rollback DROP TABLE IF EXISTS channels; From 8c52d0eb284b0b095e62883eedfd815f9ecd0e6a Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:24:53 +0100 Subject: [PATCH 08/24] Implement videos table. --- .../changelog/version/0-1-init-crdb.sql | 19 +++++++++++++++++++ .../changelog/version/0-1-init-pg.sql | 19 +++++++++++++++++++ .../resources/changelog/version/0-1-init.sql | 2 -- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index ec15eae..0c5e646 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -1,3 +1,22 @@ CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC) STORING (password, username); --rollback DROP INDEX IF EXISTS users_session_id_idx; + +CREATE TABLE IF NOT EXISTS videos ( + id VARCHAR(16) NOT NULL UNIQUE, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploaded INT8 NULL, + views INT8 NULL, + uploader_id VARCHAR(24) NOT NULL, + is_short BOOL NOT NULL DEFAULT false, + CONSTRAINT videos_pkey PRIMARY KEY (id ASC, uploader_id ASC) USING HASH, + CONSTRAINT fk_videos_channels_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id), + INDEX videos_id_idx (id ASC), + INDEX video_uploaded_idx (uploaded ASC) USING HASH, + INDEX video_uploader_id_idx (uploader_id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short), + UNIQUE INDEX videos_id_key (id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short) +); + +--rollback DROP TABLE IF EXISTS videos; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 610e4c2..009add6 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -1,3 +1,22 @@ CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC); --rollback DROP INDEX IF EXISTS users_session_id_idx; + +CREATE TABLE IF NOT EXISTS videos ( + id VARCHAR(16) NOT NULL UNIQUE, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploaded INT8 NULL, + views INT8 NULL, + uploader_id VARCHAR(24) NOT NULL, + is_short BOOL NOT NULL DEFAULT false, + CONSTRAINT videos_pkey PRIMARY KEY (id, uploader_id), + CONSTRAINT fk_videos_channels_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS videos_id_idx ON videos (id ASC); +CREATE INDEX IF NOT EXISTS video_uploaded_idx ON videos (uploaded ASC); +CREATE INDEX IF NOT EXISTS video_uploader_id_idx ON videos (uploader_id ASC); + +--rollback DROP TABLE IF EXISTS videos; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index 6e6eb01..126e42f 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -10,7 +10,6 @@ DROP INDEX IF EXISTS users_id_idx; CREATE INDEX IF NOT EXISTS username_idx ON users (username ASC); ---rollback DROP INDEX IF EXISTS username_idx; --rollback DROP TABLE IF EXISTS users; CREATE TABLE IF NOT EXISTS channels ( @@ -23,5 +22,4 @@ CREATE TABLE IF NOT EXISTS channels ( CREATE INDEX IF NOT EXISTS channels_uploader_idx ON channels (uploader ASC); ---rollback DROP INDEX IF EXISTS channels_uploader_idx; --rollback DROP TABLE IF EXISTS channels; From b0a97faf4f60b74fe32edc982e6f938b244c088d Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:32:26 +0100 Subject: [PATCH 09/24] Add users_subscribed table. --- .../resources/changelog/version/0-1-init-crdb.sql | 11 +++++++++++ src/main/resources/changelog/version/0-1-init-pg.sql | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index 0c5e646..395cf02 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -20,3 +20,14 @@ CREATE TABLE IF NOT EXISTS videos ( ); --rollback DROP TABLE IF EXISTS videos; + +CREATE TABLE IF NOT EXISTS users_subscribed ( + subscriber INT8 NOT NULL, + channel VARCHAR(24) NOT NULL, + CONSTRAINT users_subscribed_pkey PRIMARY KEY (subscriber ASC, channel ASC) USING HASH, + CONSTRAINT fk_subscriber_users FOREIGN KEY (subscriber) REFERENCES users(id), + INDEX users_subscribed_subscriber_idx (subscriber ASC), + INDEX users_subscribed_channel_idx (channel ASC) +); + +--rollback DROP TABLE IF EXISTS users_subscribed; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 009add6..8362d82 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -20,3 +20,15 @@ CREATE INDEX IF NOT EXISTS video_uploaded_idx ON videos (uploaded ASC); CREATE INDEX IF NOT EXISTS video_uploader_id_idx ON videos (uploader_id ASC); --rollback DROP TABLE IF EXISTS videos; + +CREATE TABLE IF NOT EXISTS users_subscribed ( + subscriber INT8 NOT NULL, + channel VARCHAR(24) NOT NULL, + CONSTRAINT users_subscribed_pkey PRIMARY KEY (subscriber, channel), + CONSTRAINT fk_subscriber_users FOREIGN KEY (subscriber) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS users_subscribed_subscriber_idx ON users_subscribed (subscriber ASC); +CREATE INDEX IF NOT EXISTS users_subscribed_channel_idx ON users_subscribed (channel ASC); + +--rollback DROP TABLE IF EXISTS users_subscribed; From b3c06354f7e5cfc6e3b4c6955ea5941aa8d97473 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:43:07 +0100 Subject: [PATCH 10/24] Add pubsub table. --- src/main/resources/changelog/version/0-1-init-crdb.sql | 4 ++++ src/main/resources/changelog/version/0-1-init-pg.sql | 4 ++++ src/main/resources/changelog/version/0-1-init.sql | 10 ++++++++++ 3 files changed, 18 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index 395cf02..2f2d037 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -31,3 +31,7 @@ CREATE TABLE IF NOT EXISTS users_subscribed ( ); --rollback DROP TABLE IF EXISTS users_subscribed; + +CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC) USING HASH; + +--rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 8362d82..7694b7b 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -32,3 +32,7 @@ CREATE INDEX IF NOT EXISTS users_subscribed_subscriber_idx ON users_subscribed ( CREATE INDEX IF NOT EXISTS users_subscribed_channel_idx ON users_subscribed (channel ASC); --rollback DROP TABLE IF EXISTS users_subscribed; + +CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC); + +--rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index 126e42f..8caa6e1 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -23,3 +23,13 @@ CREATE TABLE IF NOT EXISTS channels ( CREATE INDEX IF NOT EXISTS channels_uploader_idx ON channels (uploader ASC); --rollback DROP TABLE IF EXISTS channels; + +CREATE TABLE IF NOT EXISTS pubsub ( + id VARCHAR(24) NOT NULL, + subbed_at INT8 NULL, + CONSTRAINT pubsub_pkey PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS pubsub_id_idx ON pubsub (id ASC); + +--rollback DROP TABLE IF EXISTS pubsub; From dc7801b4d39c77f790f22a659a815d07208ef9f6 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:55:03 +0100 Subject: [PATCH 11/24] Add playlists table. --- .../resources/changelog/version/0-1-init-crdb.sql | 6 ++++++ .../resources/changelog/version/0-1-init-pg.sql | 6 ++++++ src/main/resources/changelog/version/0-1-init.sql | 13 +++++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index 2f2d037..06f0f39 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -35,3 +35,9 @@ CREATE TABLE IF NOT EXISTS users_subscribed ( CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC) USING HASH; --rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; + +CREATE INDEX IF NOT EXISTS playlists_playlist_id_idx ON playlists (playlist_id ASC) STORING (name, short_description, thumbnail, owner); +CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC) STORING (name, short_description, thumbnail, playlist_id); + +--rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; +--rollback DROP INDEX IF EXISTS playlists_owner_idx; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 7694b7b..6920c2e 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -36,3 +36,9 @@ CREATE INDEX IF NOT EXISTS users_subscribed_channel_idx ON users_subscribed (cha CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC); --rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; + +CREATE INDEX IF NOT EXISTS playlists_playlist_id_idx ON playlists (playlist_id ASC); +CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC); + +--rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; +--rollback DROP INDEX IF EXISTS playlists_owner_idx; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index 8caa6e1..f04c7f4 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -33,3 +33,16 @@ CREATE TABLE IF NOT EXISTS pubsub ( CREATE INDEX IF NOT EXISTS pubsub_id_idx ON pubsub (id ASC); --rollback DROP TABLE IF EXISTS pubsub; + +CREATE TABLE IF NOT EXISTS playlists ( + id SERIAL NOT NULL, + name VARCHAR(200) NULL, + playlist_id UUID NOT NULL UNIQUE DEFAULT gen_random_uuid(), + short_description VARCHAR(100) NULL, + thumbnail VARCHAR(300) NULL, + owner INT8 NOT NULL, + CONSTRAINT playlists_pkey PRIMARY KEY (id), + CONSTRAINT fk_playlists_owner FOREIGN KEY (owner) REFERENCES users(id) +); + +--rollback DROP TABLE IF EXISTS playlists; From 96819a51c6ab26b68a51778afc686a6c241af711 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 08:58:03 +0100 Subject: [PATCH 12/24] Update videos id size to 11. --- src/main/resources/changelog/version/0-1-init-crdb.sql | 2 +- src/main/resources/changelog/version/0-1-init-pg.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index 06f0f39..6dc280c 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -3,7 +3,7 @@ CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC) STORIN --rollback DROP INDEX IF EXISTS users_session_id_idx; CREATE TABLE IF NOT EXISTS videos ( - id VARCHAR(16) NOT NULL UNIQUE, + id VARCHAR(11) NOT NULL UNIQUE, duration INT8 NULL, thumbnail VARCHAR(400) NULL, title VARCHAR(120) NULL, diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 6920c2e..2b4499f 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -3,7 +3,7 @@ CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC); --rollback DROP INDEX IF EXISTS users_session_id_idx; CREATE TABLE IF NOT EXISTS videos ( - id VARCHAR(16) NOT NULL UNIQUE, + id VARCHAR(11) NOT NULL UNIQUE, duration INT8 NULL, thumbnail VARCHAR(400) NULL, title VARCHAR(120) NULL, From be7f54617dae202cf1aee178cf76e1d199a203aa Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:01:00 +0100 Subject: [PATCH 13/24] Add playlist_videos table. --- src/main/resources/changelog/version/0-1-init.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index f04c7f4..4027b4c 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -46,3 +46,18 @@ CREATE TABLE IF NOT EXISTS playlists ( ); --rollback DROP TABLE IF EXISTS playlists; + +CREATE TABLE IF NOT EXISTS playlist_videos ( + id VARCHAR(11) NOT NULL, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploader_id VARCHAR(24) NOT NULL, + CONSTRAINT playlist_videos_pkey PRIMARY KEY (id), + CONSTRAINT fk_playlist_video_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id) +); + +CREATE INDEX IF NOT EXISTS playlist_videos_id_idx ON playlist_videos (id ASC); +CREATE INDEX IF NOT EXISTS playlist_videos_uploader_id_idx ON playlist_videos (uploader_id ASC); + +--rollback DROP TABLE IF EXISTS playlist_videos; From 8b8b8af99bc1ea5e15e3ef815659b5c7c3919cf9 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:05:29 +0100 Subject: [PATCH 14/24] Add playlists_videos_ids table. --- src/main/resources/changelog/version/0-1-init.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index 4027b4c..3abf26f 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -61,3 +61,16 @@ CREATE INDEX IF NOT EXISTS playlist_videos_id_idx ON playlist_videos (id ASC); CREATE INDEX IF NOT EXISTS playlist_videos_uploader_id_idx ON playlist_videos (uploader_id ASC); --rollback DROP TABLE IF EXISTS playlist_videos; + +CREATE TABLE IF NOT EXISTS playlists_videos_ids ( + playlist_id INT8 NOT NULL, + videos_id VARCHAR(11) NOT NULL, + videos_order INT8 NOT NULL, + CONSTRAINT playlists_videos_ids_pkey PRIMARY KEY (playlist_id, videos_order), + CONSTRAINT fk_playlists_videos_video_id_playlist_video FOREIGN KEY (videos_id) REFERENCES playlist_videos(id), + CONSTRAINT fk_playlists_videos_playlist_id_playlist FOREIGN KEY (playlist_id) REFERENCES playlists(id) +); + +CREATE INDEX IF NOT EXISTS playlists_videos_ids_playlist_id_idx ON playlists_videos_ids (playlist_id ASC); + +--rollback DROP TABLE IF EXISTS playlists_videos_ids; From 1d08fda665782d8ae71e4bcbf34a1915b5d9bf63 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:16:01 +0100 Subject: [PATCH 15/24] Use BIGSERIAL instead of SERIAL --- src/main/resources/changelog/version/0-1-init.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index 3abf26f..eef4ce1 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS users ( - id SERIAL NOT NULL, + id BIGSERIAL NOT NULL, password TEXT NULL, session_id VARCHAR(36) NULL, username VARCHAR(24) NULL UNIQUE, @@ -35,7 +35,7 @@ CREATE INDEX IF NOT EXISTS pubsub_id_idx ON pubsub (id ASC); --rollback DROP TABLE IF EXISTS pubsub; CREATE TABLE IF NOT EXISTS playlists ( - id SERIAL NOT NULL, + id BIGSERIAL NOT NULL, name VARCHAR(200) NULL, playlist_id UUID NOT NULL UNIQUE DEFAULT gen_random_uuid(), short_description VARCHAR(100) NULL, From c8f7e528206167523027ba044f4af4bd9c8d1fb8 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:21:08 +0100 Subject: [PATCH 16/24] Fix videos_order type. --- src/main/resources/changelog/version/0-1-init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index eef4ce1..ce2689e 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -65,7 +65,7 @@ CREATE INDEX IF NOT EXISTS playlist_videos_uploader_id_idx ON playlist_videos (u CREATE TABLE IF NOT EXISTS playlists_videos_ids ( playlist_id INT8 NOT NULL, videos_id VARCHAR(11) NOT NULL, - videos_order INT8 NOT NULL, + videos_order INT4 NOT NULL, CONSTRAINT playlists_videos_ids_pkey PRIMARY KEY (playlist_id, videos_order), CONSTRAINT fk_playlists_videos_video_id_playlist_video FOREIGN KEY (videos_id) REFERENCES playlist_videos(id), CONSTRAINT fk_playlists_videos_playlist_id_playlist FOREIGN KEY (playlist_id) REFERENCES playlists(id) From 657e63332e014e0c4292e77499c59eb0dfd039a7 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:30:53 +0100 Subject: [PATCH 17/24] Add unauthenticated_subscriptions table. --- .../resources/changelog/version/0-1-init-crdb.sql | 4 ++++ src/main/resources/changelog/version/0-1-init-pg.sql | 4 ++++ src/main/resources/changelog/version/0-1-init.sql | 11 +++++++++++ 3 files changed, 19 insertions(+) diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql index 6dc280c..20b5879 100644 --- a/src/main/resources/changelog/version/0-1-init-crdb.sql +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -41,3 +41,7 @@ CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC) STORING --rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; --rollback DROP INDEX IF EXISTS playlists_owner_idx; + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_id_idx ON unauthenticated_subscriptions (id ASC) USING HASH STORING (subscribed_at); + +--rollback DROP INDEX IF EXISTS unauthenticated_subscriptions_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql index 2b4499f..1fc4351 100644 --- a/src/main/resources/changelog/version/0-1-init-pg.sql +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -42,3 +42,7 @@ CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC); --rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; --rollback DROP INDEX IF EXISTS playlists_owner_idx; + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_id_idx ON unauthenticated_subscriptions (id ASC); + +--rollback DROP INDEX IF EXISTS unauthenticated_subscriptions_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql index ce2689e..0accbb9 100644 --- a/src/main/resources/changelog/version/0-1-init.sql +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -74,3 +74,14 @@ CREATE TABLE IF NOT EXISTS playlists_videos_ids ( CREATE INDEX IF NOT EXISTS playlists_videos_ids_playlist_id_idx ON playlists_videos_ids (playlist_id ASC); --rollback DROP TABLE IF EXISTS playlists_videos_ids; + +CREATE TABLE IF NOT EXISTS unauthenticated_subscriptions ( + id VARCHAR(24) NOT NULL, + subscribed_at INT8 NOT NULL, + CONSTRAINT unauthenticated_subscriptions_pkey PRIMARY KEY (id), + CONSTRAINT fk_unauthenticated_subscriptions_id_channels FOREIGN KEY (id) REFERENCES channels(uploader_id) +); + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_subscribed_at_idx ON unauthenticated_subscriptions (subscribed_at ASC); + +--rollback DROP TABLE IF EXISTS unauthenticated_subscriptions; From 66abab9389ec8020d40e93ee174aed80bee547a9 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 09:52:59 +0100 Subject: [PATCH 18/24] Add pgcrypto extension on yugabytedb. --- src/main/resources/changelog/version/0-1-init-yb.sql | 2 ++ src/main/resources/changelog/version/0-init.xml | 1 + 2 files changed, 3 insertions(+) create mode 100644 src/main/resources/changelog/version/0-1-init-yb.sql diff --git a/src/main/resources/changelog/version/0-1-init-yb.sql b/src/main/resources/changelog/version/0-1-init-yb.sql new file mode 100644 index 0000000..7b3c88b --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init-yb.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION pgcrypto; +--rollback DROP EXTENSION IF EXISTS pgcrypto; diff --git a/src/main/resources/changelog/version/0-init.xml b/src/main/resources/changelog/version/0-init.xml index 68b90cc..d9f7b4b 100644 --- a/src/main/resources/changelog/version/0-init.xml +++ b/src/main/resources/changelog/version/0-init.xml @@ -7,6 +7,7 @@ + From 28a1a481c7d95de382c1285ece832655ab1964cc Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:03:05 +0100 Subject: [PATCH 19/24] Register YugabyteDB extension. --- src/main/java/me/kavin/piped/utils/LiquibaseHelper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java index f412ff8..a97d09c 100644 --- a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java @@ -26,6 +26,9 @@ public class LiquibaseHelper { // ensure postgres driver is loaded DriverManager.registerDriver(new org.postgresql.Driver()); + // register YugabyteDB database + DatabaseFactory.getInstance().register(new liquibase.ext.yugabytedb.database.YugabyteDBDatabase()); + Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(DriverManager.getConnection(url, username, password))); try (Liquibase liquibase = new Liquibase("changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), database)) { From 1175a1c6c311b5aa9d4fc5f8528f745c3f1818a4 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:24:36 +0100 Subject: [PATCH 20/24] Attempt to run create extension before. --- .../changelog/version/{0-1-init-yb.sql => 0-0-init-yb.sql} | 0 src/main/resources/changelog/version/0-init.xml | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) rename src/main/resources/changelog/version/{0-1-init-yb.sql => 0-0-init-yb.sql} (100%) diff --git a/src/main/resources/changelog/version/0-1-init-yb.sql b/src/main/resources/changelog/version/0-0-init-yb.sql similarity index 100% rename from src/main/resources/changelog/version/0-1-init-yb.sql rename to src/main/resources/changelog/version/0-0-init-yb.sql diff --git a/src/main/resources/changelog/version/0-init.xml b/src/main/resources/changelog/version/0-init.xml index d9f7b4b..649566a 100644 --- a/src/main/resources/changelog/version/0-init.xml +++ b/src/main/resources/changelog/version/0-init.xml @@ -4,10 +4,13 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + + + + - From e32ffb16a15c32152d5782295a16f6606c71d2fd Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:55:31 +0100 Subject: [PATCH 21/24] Run Liquibase migrations blocking before startup. --- .github/workflows/docker-build-test.yml | 8 +++++++- src/main/java/me/kavin/piped/Main.java | 14 +++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index f58a6a4..34ce3bb 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -23,6 +23,12 @@ jobs: - Dockerfile.azul.ci - Dockerfile.openj9.ci - Dockerfile.graalvm-jvm.ci + include: + - sleep: 20 + - docker-compose-file: testing/docker-compose.cockroachdb.yml + sleep: 30 + - docker-compose-file: testing/docker-compose.yugabytedb.yml + sleep: 120 fail-fast: false steps: - uses: actions/checkout@v3 @@ -45,7 +51,7 @@ jobs: file: ${{ matrix.dockerfile }} tags: 1337kavin/piped:latest - name: Start Docker-Compose services - run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep 20 + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} - name: Run tests run: ./testing/api-test.sh - name: Collect services logs diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index 15e982c..bf6ab03 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -46,6 +46,13 @@ public class Main { Injector.useSpecializer(); + try { + LiquibaseHelper.init(); + } catch (Exception e) { + ExceptionHandler.handle(e); + System.exit(1); + } + Multithreading.runAsync(() -> new Thread(new SyncRunner( new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(), MATRIX_SERVER, @@ -69,13 +76,6 @@ public class Main { } }).start(); - try { - LiquibaseHelper.init(); - } catch (Exception e) { - ExceptionHandler.handle(e); - System.exit(1); - } - try (Session ignored = DatabaseSessionFactory.createSession()) { System.out.println("Database connection is ready!"); } catch (Throwable t) { From 261fba8238f83a1f33533da098b5d5321ee4f3d5 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:05:05 +0100 Subject: [PATCH 22/24] Drop hsqldb support. --- .github/workflows/docker-build-test.yml | 1 - build.gradle | 1 - testing/config.hsqldb.properties | 18 ------------------ testing/docker-compose.hsqldb.yml | 8 -------- 4 files changed, 28 deletions(-) delete mode 100644 testing/config.hsqldb.properties delete mode 100644 testing/docker-compose.hsqldb.yml diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index 34ce3bb..1ddc3fb 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -15,7 +15,6 @@ jobs: matrix: docker-compose-file: - docker-compose.yml - - testing/docker-compose.hsqldb.yml - testing/docker-compose.cockroachdb.yml - testing/docker-compose.yugabytedb.yml dockerfile: diff --git a/build.gradle b/build.gradle index 2d8dae0..bde430e 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation 'io.activej:activej-boot:5.5' implementation 'io.activej:activej-specializer:5.5' implementation 'io.activej:activej-launchers-http:5.5' - implementation 'org.hsqldb:hsqldb:2.7.2' implementation 'org.postgresql:postgresql:42.6.0' implementation 'org.hibernate:hibernate-core:6.2.7.Final' implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' diff --git a/testing/config.hsqldb.properties b/testing/config.hsqldb.properties deleted file mode 100644 index 4c8ebe2..0000000 --- a/testing/config.hsqldb.properties +++ /dev/null @@ -1,18 +0,0 @@ -# The port to Listen on. -PORT: 8080 - -# Proxy -PROXY_PART: https://pipedproxy-ams.kavin.rocks - -# Public API URL -API_URL: https://pipedapi.kavin.rocks - -# Public Frontend URL -FRONTEND_URL: https://piped.video - -# Hibernate properties -hibernate.connection.url: jdbc:hsqldb:mem:memdb;sql.syntax_pgs=true -hibernate.connection.driver_class: org.hsqldb.jdbcDriver -hibernate.dialect: org.hibernate.dialect.HSQLDialect -hibernate.connection.username: piped -hibernate.connection.password: changeme diff --git a/testing/docker-compose.hsqldb.yml b/testing/docker-compose.hsqldb.yml deleted file mode 100644 index 31f4070..0000000 --- a/testing/docker-compose.hsqldb.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - piped: - image: 1337kavin/piped:latest - restart: unless-stopped - ports: - - "127.0.0.1:8080:8080" - volumes: - - ./config.hsqldb.properties:/app/config.properties From 2864af856b92eb01380330da480504d008ffe1c9 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:26:52 +0100 Subject: [PATCH 23/24] Use common fat-build workflow. --- .github/workflows/docker-build-test.yml | 16 +++++++--------- .github/workflows/docker-build.yml | 16 +++++++--------- .github/workflows/fat-build.yml | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/fat-build.yml diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index 1ddc3fb..07913c9 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -9,8 +9,12 @@ on: pull_request: jobs: + build-jdk: + uses: ./.github/workflows/fat-build.yml + build-test: runs-on: ubuntu-latest + needs: build-jdk strategy: matrix: docker-compose-file: @@ -31,17 +35,11 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: piped.jar - name: Create Version File run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION - - name: set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: temurin - cache: "gradle" - - name: Run Build - run: ./gradlew shadowJar - - run: mv build/libs/piped-*-all.jar piped.jar - name: Build Image Locally uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3327e48..ceb86fa 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -8,7 +8,11 @@ on: - master jobs: + build-jdk: + uses: ./.github/workflows/fat-build.yml + build-docker: + needs: build-jdk runs-on: ubuntu-latest strategy: matrix: @@ -23,17 +27,11 @@ jobs: dockerfile: ./Dockerfile.graalvm-jvm.ci steps: - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: piped.jar - name: Create Version File run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION - - name: set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: temurin - cache: "gradle" - - name: Run Build - run: ./gradlew shadowJar - - run: mv build/libs/piped-*-all.jar piped.jar - name: Set up QEMU uses: docker/setup-qemu-action@v2 with: diff --git a/.github/workflows/fat-build.yml b/.github/workflows/fat-build.yml new file mode 100644 index 0000000..3c54d07 --- /dev/null +++ b/.github/workflows/fat-build.yml @@ -0,0 +1,24 @@ +name: Fat JAR Build + +on: + workflow_call: + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: temurin + cache: "gradle" + - name: Run Build + run: ./gradlew shadowJar + - run: mv build/libs/piped-*-all.jar piped.jar + - uses: actions/upload-artifact@v3 + with: + name: piped.jar + path: piped.jar From 5fb08f66c5d703405ffce4ba3dc5dd2c9025f589 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Mon, 14 Aug 2023 18:26:14 +0100 Subject: [PATCH 24/24] actions: add action to test migrations --- .../docker-migrations-build-test.yml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/docker-migrations-build-test.yml diff --git a/.github/workflows/docker-migrations-build-test.yml b/.github/workflows/docker-migrations-build-test.yml new file mode 100644 index 0000000..293b2e6 --- /dev/null +++ b/.github/workflows/docker-migrations-build-test.yml @@ -0,0 +1,82 @@ +name: Docker-Compose Build and Test Migration + +on: + pull_request: + paths: + - "src/main/resources/changelog/**" + - "src/main/java/me/kavin/piped/utils/obj/db/**" + +jobs: + build-new: + uses: ./.github/workflows/fat-build.yml + build-old: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.base.sha }} + - name: set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: temurin + cache: "gradle" + - name: Run Build + run: ./gradlew shadowJar + - run: mv build/libs/piped-*-all.jar piped.jar + - uses: actions/upload-artifact@v3 + with: + name: piped-old.jar + path: piped.jar + + docker-build-test: + needs: [ build-new, build-old ] + runs-on: ubuntu-latest + strategy: + matrix: + docker-compose-file: + - docker-compose.yml + - testing/docker-compose.cockroachdb.yml + - testing/docker-compose.yugabytedb.yml + dockerfile: + - Dockerfile.ci + include: + - sleep: 20 + - docker-compose-file: testing/docker-compose.cockroachdb.yml + sleep: 30 + - docker-compose-file: testing/docker-compose.yugabytedb.yml + sleep: 120 + fail-fast: false + steps: + - uses: actions/checkout@v3 + - run: echo "unknown" > VERSION + - uses: actions/download-artifact@v3 + with: + name: piped-old.jar + - name: Build Old Image Locally + uses: docker/build-push-action@v4 + with: + context: . + load: true + file: ${{ matrix.dockerfile }} + tags: 1337kavin/piped:latest + - name: Start Docker-Compose services + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} + - run: rm piped.jar + - uses: actions/download-artifact@v3 + with: + name: piped.jar + - name: Build New Image Locally + uses: docker/build-push-action@v4 + with: + context: . + load: true + file: ${{ matrix.dockerfile }} + tags: 1337kavin/piped:latest + - name: Start Docker-Compose services + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} + - name: Run tests + run: ./testing/api-test.sh + - name: Collect services logs + if: failure() + run: docker-compose -f ${{ matrix.docker-compose-file }} logs