jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(ns-files): add a metadata layer on top for better performance & versioned ns files

Browse Source 
part of https://github.com/kestra-io/kestra/issues/5617
This commit is contained in:
brian.mulier
2025-10-31 16:59:31 +01:00
committed by brian-mulier-p
parent d20f7039c7
commit 682d258e7b
57 changed files with 2369 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
jdbc-h2/src/main/java/io/kestra/repository/h2/H2NamespaceFileMetadataRepository.java Normal file
View File

@@ -0,0 +1,23 @@
package io.kestra.repository.h2;
import io.kestra.core.models.namespaces.files.NamespaceFileMetadata;
import io.kestra.jdbc.repository.AbstractJdbcNamespaceFileMetadataRepository;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import org.jooq.Condition;
@Singleton
@H2RepositoryEnabled
public class H2NamespaceFileMetadataRepository extends AbstractJdbcNamespaceFileMetadataRepository {
@Inject
public H2NamespaceFileMetadataRepository(@Named("namespaceFileMetadata") H2Repository<NamespaceFileMetadata> repository) {
super(repository);
}
@Override
protected Condition findCondition(String query) {
return H2NamespaceFileMetadataRepositoryService.findCondition(jdbcRepository, query);
}
}

21
jdbc-h2/src/main/java/io/kestra/repository/h2/H2NamespaceFileMetadataRepositoryService.java Normal file
View File

@@ -0,0 +1,21 @@
package io.kestra.repository.h2;
import io.kestra.core.models.namespaces.files.NamespaceFileMetadata;
import io.kestra.jdbc.AbstractJdbcRepository;
import org.jooq.Condition;
import org.jooq.impl.DSL;
import java.util.ArrayList;
import java.util.List;
public abstract class H2NamespaceFileMetadataRepositoryService {
public static Condition findCondition(AbstractJdbcRepository<NamespaceFileMetadata> jdbcRepository, String query) {
List<Condition> conditions = new ArrayList<>();
if (query != null) {
conditions.add(jdbcRepository.fullTextCondition(List.of("fulltext"), query));
}
return conditions.isEmpty() ? DSL.trueCondition() : DSL.and(conditions);
}
}

1
jdbc-h2/src/main/resources/migrations/h2/V1_49__add_created_to_kv_metadata.sql Normal file
View File

@@ -0,0 +1 @@
ALTER TABLE kv_metadata ALTER COLUMN "created" TIMESTAMP NOT NULL GENERATED ALWAYS AS (PARSEDATETIME(LEFT(COALESCE(JQ_STRING("value", '.created'), JQ_STRING("value", '.updated')), 23) || '+00:00', 'yyyy-MM-dd''T''HH:mm:ss.SSSXXX'))

21
jdbc-h2/src/main/resources/migrations/h2/V1_50__ns_files_metadata.sql Normal file
View File

@@ -0,0 +1,21 @@
CREATE TABLE IF NOT EXISTS namespace_file_metadata (
"key" VARCHAR(768) NOT NULL PRIMARY KEY,
"value" TEXT NOT NULL,
"tenant_id" VARCHAR(250) GENERATED ALWAYS AS (JQ_STRING("value", '.tenantId')),
"namespace" VARCHAR(150) NOT NULL GENERATED ALWAYS AS (JQ_STRING("value", '.namespace')),
"path" VARCHAR(350) NOT NULL GENERATED ALWAYS AS (JQ_STRING("value", '.path')),
"parent_path" VARCHAR(350) GENERATED ALWAYS AS (JQ_STRING("value", '.parentPath')),
"version" INT NOT NULL GENERATED ALWAYS AS (JQ_INTEGER("value", '.version')),
"last" BOOL NOT NULL GENERATED ALWAYS AS (JQ_BOOLEAN("value", '.last')),
"size" BIGINT NOT NULL GENERATED ALWAYS AS (JQ_LONG("value", '.size')),
"created" TIMESTAMP NOT NULL GENERATED ALWAYS AS (PARSEDATETIME(LEFT(JQ_STRING("value", '.created'), 23) || '+00:00', 'yyyy-MM-dd''T''HH:mm:ss.SSSXXX')),
"updated" TIMESTAMP NOT NULL GENERATED ALWAYS AS (PARSEDATETIME(LEFT(JQ_STRING("value", '.updated'), 23) || '+00:00', 'yyyy-MM-dd''T''HH:mm:ss.SSSXXX')),
"deleted" BOOL NOT NULL GENERATED ALWAYS AS (JQ_BOOLEAN("value", '.deleted')),
"fulltext" TEXT NOT NULL GENERATED ALWAYS AS (JQ_STRING("value", '.path'))
);
CREATE INDEX IF NOT EXISTS ix_last_deleted_tenant_namespace_path_version ON namespace_file_metadata ("last", "deleted", "tenant_id", "namespace", "path", "version");
CREATE INDEX IF NOT EXISTS ix_last_deleted_tenant_namespace_path ON namespace_file_metadata ("last", "deleted", "tenant_id", "namespace", "path");
CREATE INDEX IF NOT EXISTS ix_last_deleted_tenant_namespace_parent_path ON namespace_file_metadata ("last", "deleted", "tenant_id", "namespace", "parent_path");
CREATE INDEX IF NOT EXISTS ix_last_deleted_tenant_namespace_version ON namespace_file_metadata ("last", "deleted", "tenant_id", "namespace", "version");
CREATE INDEX IF NOT EXISTS ix_last_deleted_tenant_path_version ON namespace_file_metadata ("last", "deleted", "tenant_id", "path", "version");

7
jdbc-h2/src/test/java/io/kestra/repository/h2/H2NamespaceFileMetadataRepositoryTest.java Normal file
View File

@@ -0,0 +1,7 @@
package io.kestra.repository.h2;
import io.kestra.jdbc.repository.AbstractJdbcNamespaceFileMetadataRepositoryTest;
public class H2NamespaceFileMetadataRepositoryTest extends AbstractJdbcNamespaceFileMetadataRepositoryTest {
}