jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(jdbc): implementation of trigger repository

Browse Source
This commit is contained in:
Ludovic DEHON
2022-05-21 21:21:16 +02:00
parent 454a603ff7
commit 7e16b718be
42 changed files with 1426 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
jdbc-mysql/build.gradle
View File

@@ -11,4 +11,5 @@ dependencies {
testImplementation project(':core').sourceSets.test.output
testImplementation project(':jdbc').sourceSets.test.output
testImplementation project(':runner-memory')
testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1'
}

17
jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlExecutionRepository.java Normal file
View File

@@ -0,0 +1,17 @@
package io.kestra.repository.mysql;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.repositories.ExecutionRepositoryInterface;
import io.kestra.jdbc.repository.AbstractExecutionRepository;
import io.micronaut.context.ApplicationContext;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@Singleton
@MysqlRepositoryEnabled
public class MysqlExecutionRepository extends AbstractExecutionRepository implements ExecutionRepositoryInterface {
@Inject
public MysqlExecutionRepository(ApplicationContext applicationContext) {
super(new MysqlRepository<>(Execution.class, applicationContext), applicationContext);
}
}

58
jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlFlowRepository.java
View File

@@ -8,10 +8,7 @@ import io.micronaut.context.ApplicationContext;
import io.micronaut.data.model.Pageable;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.jooq.Field;
import org.jooq.Record;
import org.jooq.Record1;
import org.jooq.SelectConditionStep;
import org.jooq.*;
import org.jooq.impl.DSL;
import java.util.ArrayList;
@@ -28,15 +25,14 @@ public class MysqlFlowRepository extends AbstractFlowRepository {
}
@SuppressWarnings("unchecked")
private <R extends Record> SelectConditionStep<R> fullTextSelect(List<Field<Object>> field) {
private <R extends Record> SelectConditionStep<R> fullTextSelect(DSLContext context, List<Field<Object>> field) {
ArrayList<Field<Object>> fields = new ArrayList<>(Collections.singletonList(DSL.field("value")));
if (field != null) {
fields.addAll(field);
}
return (SelectConditionStep<R>) this.jdbcRepository
.getDslContext()
return (SelectConditionStep<R>) context
.select(fields)
.hint("SQL_CALC_FOUND_ROWS")
.from(lastRevision(false))
@@ -49,30 +45,44 @@ public class MysqlFlowRepository extends AbstractFlowRepository {
}
public ArrayListTotal<Flow> find(String query, Pageable pageable) {
SelectConditionStep<Record1<Object>> select = this.fullTextSelect(Collections.emptyList());
return this.jdbcRepository
.getDslContext()
.transactionResult(configuration -> {
DSLContext context = DSL.using(configuration);
if (query != null) {
select.and(this.jdbcRepository.fullTextCondition(Arrays.asList("namespace", "id"), query));
}
SelectConditionStep<Record1<Object>> select = this.fullTextSelect(context, Collections.emptyList());
if (query != null) {
select.and(this.jdbcRepository.fullTextCondition(Arrays.asList("namespace", "id"), query));
}
return this.jdbcRepository.fetchPage(context, select, pageable);
});
return this.jdbcRepository.fetchPage(select, pageable);
}
@Override
public ArrayListTotal<SearchResult<Flow>> findSourceCode(String query, Pageable pageable) {
SelectConditionStep<Record> select = this.fullTextSelect(Collections.singletonList(DSL.field("source_code")));
return this.jdbcRepository
.getDslContext()
.transactionResult(configuration -> {
DSLContext context = DSL.using(configuration);
if (query != null) {
select.and(this.jdbcRepository.fullTextCondition(Collections.singletonList("source_code"), query));
}
SelectConditionStep<Record> select = this.fullTextSelect(context, Collections.singletonList(DSL.field("source_code")));
return this.jdbcRepository.fetchPage(
select,
pageable,
record -> new SearchResult<>(
this.jdbcRepository.map(record),
this.jdbcRepository.fragments(query, record.getValue("source_code", String.class))
)
);
if (query != null) {
select.and(this.jdbcRepository.fullTextCondition(Collections.singletonList("source_code"), query));
}
return this.jdbcRepository.fetchPage(
context,
select,
pageable,
record -> new SearchResult<>(
this.jdbcRepository.map(record),
this.jdbcRepository.fragments(query, record.getValue("source_code", String.class))
)
);
});
}
}

18
jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlRepository.java
View File

@@ -5,22 +5,23 @@ import io.kestra.core.repositories.ArrayListTotal;
import io.kestra.jdbc.AbstractJdbcRepository;
import io.micronaut.context.ApplicationContext;
import io.micronaut.data.model.Pageable;
import org.jooq.Condition;
import org.jooq.Record;
import org.jooq.RecordMapper;
import org.jooq.SelectConditionStep;
import org.jooq.*;
import org.jooq.impl.DSL;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class MysqlRepository<T extends DeletedInterface> extends AbstractJdbcRepository<T> {
public class MysqlRepository<T> extends AbstractJdbcRepository<T> {
public MysqlRepository(Class<T> cls, ApplicationContext applicationContext) {
super(cls, applicationContext);
}
public Condition fullTextCondition(List<String> fields, String query) {
if (query == null || query.equals("*")) {
return DSL.trueCondition();
}
String match = Arrays
.stream(query.split("\\p{IsPunct}"))
.filter(s -> s.length() >= 3)
@@ -34,11 +35,14 @@ public class MysqlRepository<T extends DeletedInterface> extends AbstractJdbcRe
return DSL.condition("MATCH (" + String.join(", ", fields) + ") AGAINST (? IN BOOLEAN MODE)", match);
}
public <R extends Record, E> ArrayListTotal<E> fetchPage(SelectConditionStep<R> select, Pageable pageable, RecordMapper<R, E> mapper) {
public <R extends Record, E> ArrayListTotal<E> fetchPage(DSLContext context, SelectConditionStep<R> select, Pageable pageable, RecordMapper<R, E> mapper) {
List<E> map = this.pageable(select, pageable)
.fetch()
.map(mapper);
return new ArrayListTotal<>(map, dslContext.fetchOne("SELECT FOUND_ROWS()").into(Integer.class));
return dslContext.transactionResult(configuration -> new ArrayListTotal<>(
map,
DSL.using(configuration).fetchOne("SELECT FOUND_ROWS()").into(Integer.class)
));
}
}

22
jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlTemplateRepository.java
View File

@@ -1,18 +1,11 @@
package io.kestra.repository.mysql;
import io.kestra.core.models.templates.Template;
import io.kestra.core.repositories.ArrayListTotal;
import io.kestra.core.repositories.TemplateRepositoryInterface;
import io.kestra.jdbc.repository.AbstractTemplateRepository;
import io.micronaut.context.ApplicationContext;
import io.micronaut.data.model.Pageable;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.jooq.Record1;
import org.jooq.SelectConditionStep;
import org.jooq.impl.DSL;
import java.util.Arrays;
@Singleton
@MysqlRepositoryEnabled
@@ -21,19 +14,4 @@ public class MysqlTemplateRepository extends AbstractTemplateRepository implemen
public MysqlTemplateRepository(ApplicationContext applicationContext) {
super(new MysqlRepository<>(Template.class, applicationContext), applicationContext);
}
public ArrayListTotal<Template> find(String query, Pageable pageable) {
SelectConditionStep<Record1<Object>> select = this.jdbcRepository
.getDslContext()
.select(DSL.field("value"))
.hint("SQL_CALC_FOUND_ROWS")
.from(this.jdbcRepository.getTable())
.where(this.defaultFilter());
if (query != null) {
select.and(this.jdbcRepository.fullTextCondition(Arrays.asList("namespace", "id"), query));
}
return this.jdbcRepository.fetchPage(select, pageable);
}
}

15
jdbc-mysql/src/main/java/io/kestra/repository/mysql/MysqlTriggerRepository.java Normal file
View File

@@ -0,0 +1,15 @@
package io.kestra.repository.mysql;
import io.kestra.core.models.triggers.Trigger;
import io.kestra.jdbc.repository.AbstractTriggerRepository;
import io.micronaut.context.ApplicationContext;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@Singleton
@MysqlRepositoryEnabled
public class MysqlTriggerRepository extends AbstractTriggerRepository {
@Inject
public MysqlTriggerRepository(ApplicationContext applicationContext) {
super(new MysqlRepository<>(Trigger.class, applicationContext));
}}

70
jdbc-mysql/src/main/resources/migrations/mysql/V1__initial.sql
View File

@@ -1,4 +1,25 @@
CREATE TABLE ${prefix}queues (
DELIMITER //
CREATE FUNCTION PARSE_ISO8601_DURATION(duration VARCHAR(20))
RETURNS bigint
LANGUAGE SQL
CONTAINS SQL
DETERMINISTIC
BEGIN
RETURN
CASE
WHEN duration LIKE 'P%DT%H%M%.%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'P%dDT%HH%iM%s.%fS.%f'))
WHEN duration LIKE 'P%DT%H%M%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'P%dDT%HH%iM%sS.%f'))
WHEN duration LIKE 'PT%H%M%.%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%HH%iM%s.%fS.%f'))
WHEN duration LIKE 'PT%H%M%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%HH%iM%sS.%f'))
WHEN duration LIKE 'PT%M%.%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%iM%s.%fS.%f'))
WHEN duration LIKE 'PT%M%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%iM%sS.%f'))
WHEN duration LIKE 'PT%.%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%s.%fS.%f'))
WHEN duration LIKE 'PT%S' THEN TO_SECONDS(STR_TO_DATE(duration, 'PT%sS.%f'))
END;
END //
DELIMITER ;
CREATE TABLE queues (
`offset` INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
`type` ENUM(
'io.kestra.core.models.executions.Execution',
@@ -22,10 +43,10 @@ CREATE TABLE ${prefix}queues (
) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
CREATE TABLE `${prefix}flows` (
CREATE TABLE `flows` (
`key` VARCHAR(250) NOT NULL PRIMARY KEY,
`value` JSON NOT NULL,
`deleted` BOOL GENERATED ALWAYS AS (value ->> '$.deleted' = 'true') STORED NOT NULL ,
`deleted` BOOL GENERATED ALWAYS AS (value ->> '$.deleted' = 'true') STORED NOT NULL,
`id` VARCHAR(100) GENERATED ALWAYS AS (value ->> '$.id') STORED NOT NULL,
`namespace` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.namespace') STORED NOT NULL,
`revision` INT UNSIGNED GENERATED ALWAYS AS (value ->> '$.revision') STORED NOT NULL,
@@ -39,10 +60,10 @@ CREATE TABLE `${prefix}flows` (
) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
CREATE TABLE `${prefix}templates` (
CREATE TABLE `templates` (
`key` VARCHAR(250) NOT NULL PRIMARY KEY,
`value` JSON NOT NULL,
`deleted` BOOL GENERATED ALWAYS AS (value ->> '$.deleted' = 'true') STORED NOT NULL ,
`deleted` BOOL GENERATED ALWAYS AS (value ->> '$.deleted' = 'true') STORED NOT NULL,
`id` VARCHAR(100) GENERATED ALWAYS AS (value ->> '$.id') STORED NOT NULL,
`namespace` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.namespace') STORED NOT NULL,
INDEX ix_id (id),
@@ -52,3 +73,42 @@ CREATE TABLE `${prefix}templates` (
) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
CREATE TABLE `executions` (
`key` VARCHAR(250) NOT NULL PRIMARY KEY,
`value` JSON NOT NULL,
`deleted` BOOL GENERATED ALWAYS AS (value ->> '$.deleted' = 'true') STORED NOT NULL,
`id` VARCHAR(100) GENERATED ALWAYS AS (value ->> '$.id') STORED NOT NULL,
`namespace` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.namespace') STORED NOT NULL,
`flow_id` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.flowId') STORED NOT NULL,
`state_current` ENUM(
'CREATED',
'RUNNING',
'PAUSED',
'RESTARTED',
'KILLING',
'SUCCESS',
'WARNING',
'FAILED',
'KILLED'
) GENERATED ALWAYS AS (value ->> '$.state.current') STORED NOT NULL,
`state_duration` BIGINT GENERATED ALWAYS AS (value ->> '$.state.duration' * 1000) STORED NOT NULL,
`start_date` TIMESTAMP GENERATED ALWAYS AS (STR_TO_DATE(value ->> '$.state.startDate' , '%Y-%m-%dT%H:%i:%s.%fZ')) STORED NOT NULL,
INDEX ix_executions_id (id),
INDEX ix_executions_namespace (namespace),
INDEX ix_executions_flowId (flow_id),
INDEX ix_executions_state_current (state_current),
INDEX ix_executions_start_date (start_date),
INDEX ix_executions_state_duration (state_duration),
INDEX ix_executions_deleted (deleted),
FULLTEXT ix_fulltext (namespace, flow_id, id)
) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
CREATE TABLE triggers (
`key` VARCHAR(250) NOT NULL PRIMARY KEY,
`value` JSON NOT NULL,
`namespace` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.namespace') STORED NOT NULL,
`flow_id` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.flowId') STORED NOT NULL,
`trigger_id` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.triggerId') STORED NOT NULL,
INDEX ix_executions_id (namespace, flow_id, trigger_id)
) ENGINE INNODB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

16
jdbc-mysql/src/test/java/io/kestra/repository/mysql/MysqlExecutionRepositoryTest.java Normal file
View File

@@ -0,0 +1,16 @@
package io.kestra.repository.mysql;
import io.kestra.jdbc.repository.AbstractJdbcExecutionRepositoryTest;
import org.junit.jupiter.api.Test;
public class MysqlExecutionRepositoryTest extends AbstractJdbcExecutionRepositoryTest {
@Test
protected void findTaskRun() {
}
@Test
protected void taskRunsDailyStatistics() {
}
}

8
jdbc-mysql/src/test/java/io/kestra/repository/mysql/MysqlTriggerRepositoryTest.java Normal file
View File

@@ -0,0 +1,8 @@
package io.kestra.repository.mysql;
import io.kestra.jdbc.repository.AbstractJdbcTriggerRepositoryTest;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
public class MysqlTriggerRepositoryTest extends AbstractJdbcTriggerRepositoryTest {
}

15
jdbc-mysql/src/test/resources/application.yml
View File

@@ -12,8 +12,6 @@ flyway:
enabled: true
locations:
- classpath:migrations/mysql
placeholders:
prefix: ""
kestra:
queue:
@@ -22,22 +20,21 @@ kestra:
type: mysql
jdbc:
table-prefix: ""
tables:
queues:
table: "${kestra.jdbc.table-prefix}queues"
table: "queues"
flows:
table: "${kestra.jdbc.table-prefix}flows"
table: "flows"
cls: io.kestra.core.models.flows.Flow
executions:
table: "${kestra.jdbc.table-prefix}executions"
table: "executions"
cls: io.kestra.core.models.executions.Execution
templates:
table: "${kestra.jdbc.table-prefix}templates"
table: "templates"
cls: io.kestra.core.models.templates.Template
triggers:
table: "${kestra.jdbc.table-prefix}triggers"
table: "triggers"
cls: io.kestra.core.models.triggers.Trigger
logs:
table: "${kestra.jdbc.table-prefix}logs"
table: "logs"
cls: io.kestra.core.models.executions.LogEntry

1
jdbc-mysql/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker Normal file
View File

@@ -0,0 +1 @@
mock-maker-inline