diff --git a/cli/src/main/resources/application.yml b/cli/src/main/resources/application.yml
index 0477fc2593..62674f54b4 100644
--- a/cli/src/main/resources/application.yml
+++ b/cli/src/main/resources/application.yml
@@ -137,6 +137,11 @@ flyway:
# We must ignore missing migrations as we delete some wrong or not used anymore migrations
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
+ properties:
+ flyway:
+ postgresql:
+ transactional:
+ lock: false
mysql:
enabled: true
locations:
diff --git a/core/src/main/java/io/kestra/core/models/SoftDeletion.java b/core/src/main/java/io/kestra/core/models/SoftDeletable.java
similarity index 93%
rename from core/src/main/java/io/kestra/core/models/SoftDeletion.java
rename to core/src/main/java/io/kestra/core/models/SoftDeletable.java
index 1dcd799b1c..f7414db9c6 100644
--- a/core/src/main/java/io/kestra/core/models/SoftDeletion.java
+++ b/core/src/main/java/io/kestra/core/models/SoftDeletable.java
@@ -5,7 +5,7 @@ package io.kestra.core.models;
* Soft deletion is based on a deleted field that is set to true when the entity is deleted.
* Physical deletion either never occurs or occurs in a dedicated purge mechanism.
*/
-public interface SoftDeletion {
+public interface SoftDeletable {
/**
* Whether en entity is deleted or not.
*/
diff --git a/core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java b/core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java
index 670dfec91c..800c3b3665 100644
--- a/core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java
+++ b/core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java
@@ -1,7 +1,7 @@
package io.kestra.core.models.dashboards;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.dashboards.charts.Chart;
import io.kestra.core.utils.IdUtils;
@@ -26,7 +26,7 @@ import java.util.Objects;
@NoArgsConstructor
@Introspected
@ToString
-public class Dashboard implements HasUID, SoftDeletion {
+public class Dashboard implements HasUID, SoftDeletable {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
private String tenantId;
diff --git a/core/src/main/java/io/kestra/core/models/executions/Execution.java b/core/src/main/java/io/kestra/core/models/executions/Execution.java
index 9dda511ffe..2ae8c33189 100644
--- a/core/src/main/java/io/kestra/core/models/executions/Execution.java
+++ b/core/src/main/java/io/kestra/core/models/executions/Execution.java
@@ -11,7 +11,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import io.kestra.core.debug.Breakpoint;
import io.kestra.core.exceptions.InternalException;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.Label;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.Flow;
@@ -53,7 +53,7 @@ import java.util.zip.CRC32;
@AllArgsConstructor
@ToString
@EqualsAndHashCode
-public class Execution implements SoftDeletion, TenantInterface {
+public class Execution implements SoftDeletable, TenantInterface {
@With
@Hidden
diff --git a/core/src/main/java/io/kestra/core/models/executions/LogEntry.java b/core/src/main/java/io/kestra/core/models/executions/LogEntry.java
index fef1a70b64..fd01c2de4b 100644
--- a/core/src/main/java/io/kestra/core/models/executions/LogEntry.java
+++ b/core/src/main/java/io/kestra/core/models/executions/LogEntry.java
@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
-import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -22,7 +21,7 @@ import java.util.stream.Stream;
@Value
@Builder(toBuilder = true)
-public class LogEntry implements DeletedInterface, TenantInterface {
+public class LogEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -57,10 +56,6 @@ public class LogEntry implements DeletedInterface, TenantInterface {
String message;
- @NotNull
- @Builder.Default
- boolean deleted = false;
-
@Nullable
ExecutionKind executionKind;
diff --git a/core/src/main/java/io/kestra/core/models/executions/MetricEntry.java b/core/src/main/java/io/kestra/core/models/executions/MetricEntry.java
index 96fd0913fe..364c2c6f05 100644
--- a/core/src/main/java/io/kestra/core/models/executions/MetricEntry.java
+++ b/core/src/main/java/io/kestra/core/models/executions/MetricEntry.java
@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
-import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.executions.metrics.Gauge;
@@ -18,7 +17,7 @@ import jakarta.validation.constraints.Pattern;
@Value
@Builder(toBuilder = true)
-public class MetricEntry implements DeletedInterface, TenantInterface {
+public class MetricEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -54,10 +53,6 @@ public class MetricEntry implements DeletedInterface, TenantInterface {
@Nullable
Map tags;
- @NotNull
- @Builder.Default
- boolean deleted = false;
-
@Nullable
ExecutionKind executionKind;
diff --git a/core/src/main/java/io/kestra/core/models/flows/FlowInterface.java b/core/src/main/java/io/kestra/core/models/flows/FlowInterface.java
index f98cda829e..76358b7afe 100644
--- a/core/src/main/java/io/kestra/core/models/flows/FlowInterface.java
+++ b/core/src/main/java/io/kestra/core/models/flows/FlowInterface.java
@@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.Label;
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
* The base interface for FLow.
*/
@JsonDeserialize(as = GenericFlow.class)
-public interface FlowInterface extends FlowId, SoftDeletion, TenantInterface, HasUID, HasSource {
+public interface FlowInterface extends FlowId, SoftDeletable, TenantInterface, HasUID, HasSource {
Pattern YAML_REVISION_MATCHER = Pattern.compile("(?m)^revision: \\d+\n?");
diff --git a/core/src/main/java/io/kestra/core/models/kv/PersistedKvMetadata.java b/core/src/main/java/io/kestra/core/models/kv/PersistedKvMetadata.java
index 2998b4d776..a3025d04e8 100644
--- a/core/src/main/java/io/kestra/core/models/kv/PersistedKvMetadata.java
+++ b/core/src/main/java/io/kestra/core/models/kv/PersistedKvMetadata.java
@@ -1,6 +1,6 @@
package io.kestra.core.models.kv;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.kv.KVEntry;
@@ -22,7 +22,7 @@ import java.util.Optional;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
-public class PersistedKvMetadata implements SoftDeletion, TenantInterface, HasUID {
+public class PersistedKvMetadata implements SoftDeletable, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
diff --git a/core/src/main/java/io/kestra/core/models/templates/Template.java b/core/src/main/java/io/kestra/core/models/templates/Template.java
index 758daed8b5..fd73bb3ae6 100644
--- a/core/src/main/java/io/kestra/core/models/templates/Template.java
+++ b/core/src/main/java/io/kestra/core/models/templates/Template.java
@@ -7,8 +7,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
-import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.HasUID;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.validations.ManualConstraintViolation;
@@ -35,7 +35,7 @@ import jakarta.validation.constraints.Pattern;
@Introspected
@ToString
@EqualsAndHashCode
-public class Template implements SoftDeletion, TenantInterface, HasUID {
+public class Template implements SoftDeletable, TenantInterface, HasUID {
private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
@Override
diff --git a/core/src/main/java/io/kestra/core/test/TestSuite.java b/core/src/main/java/io/kestra/core/test/TestSuite.java
index 7c9f545864..8b59ee62bf 100644
--- a/core/src/main/java/io/kestra/core/test/TestSuite.java
+++ b/core/src/main/java/io/kestra/core/test/TestSuite.java
@@ -1,7 +1,7 @@
package io.kestra.core.test;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
@@ -25,7 +25,7 @@ import java.util.List;
@ToString
@EqualsAndHashCode
@TestSuiteValidation
-public class TestSuite implements HasUID, TenantInterface, SoftDeletion, HasSource {
+public class TestSuite implements HasUID, TenantInterface, SoftDeletable, HasSource {
@NotNull
@NotBlank
diff --git a/core/src/main/java/io/kestra/core/test/TestSuiteRunEntity.java b/core/src/main/java/io/kestra/core/test/TestSuiteRunEntity.java
index 42452fe88b..e854b7ac74 100644
--- a/core/src/main/java/io/kestra/core/test/TestSuiteRunEntity.java
+++ b/core/src/main/java/io/kestra/core/test/TestSuiteRunEntity.java
@@ -1,6 +1,6 @@
package io.kestra.core.test;
-import io.kestra.core.models.SoftDeletion;
+import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.test.flow.UnitTestResult;
@@ -24,7 +24,7 @@ public record TestSuiteRunEntity(
String flowId,
TestState state,
List results
-) implements SoftDeletion, TenantInterface, HasUID {
+) implements SoftDeletable, TenantInterface, HasUID {
public static TestSuiteRunEntity create(String tenantId, TestSuiteUid testSuiteUid, TestSuiteRunResult testSuiteRunResult) {
return new TestSuiteRunEntity(
diff --git a/jdbc-h2/src/main/resources/migrations/h2/V1_51__logs_metrics_deleted.sql b/jdbc-h2/src/main/resources/migrations/h2/V1_51__logs_metrics_deleted.sql
new file mode 100644
index 0000000000..f3896df83b
--- /dev/null
+++ b/jdbc-h2/src/main/resources/migrations/h2/V1_51__logs_metrics_deleted.sql
@@ -0,0 +1,22 @@
+DROP INDEX logs_execution_id;
+DROP INDEX logs_execution_id__task_id;
+DROP INDEX logs_execution_id__taskrun_id;
+DROP INDEX logs_namespace_flow;
+
+ALTER table logs drop column "deleted";
+
+CREATE INDEX IF NOT EXISTS logs_execution_id ON logs ("execution_id");
+CREATE INDEX IF NOT EXISTS logs_execution_id__task_id ON logs ("execution_id", "task_id");
+CREATE INDEX IF NOT EXISTS logs_execution_id__taskrun_id ON logs ("execution_id", "taskrun_id");
+CREATE INDEX IF NOT EXISTS logs_namespace_flow ON logs ("tenant_id", "timestamp", "level", "namespace", "flow_id");
+
+
+DROP INDEX IF EXISTS metrics_flow_id;
+DROP INDEX IF EXISTS metrics_execution_id;
+DROP INDEX IF EXISTS metrics_timestamp;
+
+ALTER TABLE metrics drop column "deleted";
+
+CREATE INDEX IF NOT EXISTS metrics_flow_id ON metrics ("tenant_id", "namespace", "flow_id");
+CREATE INDEX IF NOT EXISTS metrics_execution_id ON metrics ("execution_id");
+CREATE INDEX IF NOT EXISTS metrics_timestamp ON metrics ("tenant_id", "timestamp");
\ No newline at end of file
diff --git a/jdbc-mysql/src/main/resources/migrations/mysql/V1_52__logs_metrics_deleted.sql b/jdbc-mysql/src/main/resources/migrations/mysql/V1_52__logs_metrics_deleted.sql
new file mode 100644
index 0000000000..4ba24727d6
--- /dev/null
+++ b/jdbc-mysql/src/main/resources/migrations/mysql/V1_52__logs_metrics_deleted.sql
@@ -0,0 +1,22 @@
+ALTER TABLE logs DROP INDEX ix_execution_id;
+ALTER TABLE logs DROP INDEX ix_execution_id__task_id;
+ALTER TABLE logs DROP INDEX ix_execution_id__taskrun_id;
+ALTER TABLE logs DROP INDEX ix_namespace_flow;
+
+ALTER table logs drop column `deleted`;
+
+ALTER TABLE logs ADD INDEX ix_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE logs ADD INDEX ix_execution_id__task_id (`execution_id`, `task_id`), ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE logs ADD INDEX ix_execution_id__taskrun_id (`execution_id`, `taskrun_id`), ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE logs ADD INDEX ix_namespace_flow (`tenant_id`, `timestamp`, `level`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
+
+
+ALTER TABLE metrics DROP INDEX metrics_flow_id;
+ALTER TABLE metrics DROP INDEX ix_metrics_execution_id;
+ALTER TABLE metrics DROP INDEX metrics_timestamp;
+
+ALTER TABLE metrics drop column `deleted`;
+
+ALTER TABLE metrics ADD INDEX ix_metrics_flow_id (`tenant_id`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE metrics ADD INDEX ix_metrics_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE metrics ADD INDEX ix_metrics_timestamp (`tenant_id`, `timestamp`), ALGORITHM=INPLACE, LOCK=NONE;
\ No newline at end of file
diff --git a/jdbc-postgres/src/main/resources/migrations/postgres/V1_51__logs_metrics_deleted.sql b/jdbc-postgres/src/main/resources/migrations/postgres/V1_51__logs_metrics_deleted.sql
new file mode 100644
index 0000000000..2f323379dc
--- /dev/null
+++ b/jdbc-postgres/src/main/resources/migrations/postgres/V1_51__logs_metrics_deleted.sql
@@ -0,0 +1,13 @@
+-- Indices will be re-created by the next migration
+DROP INDEX logs_execution_id;
+DROP INDEX logs_execution_id__task_id;
+DROP INDEX logs_execution_id__taskrun_id;
+DROP INDEX logs_namespace_flow;
+
+ALTER table logs drop column "deleted";
+
+DROP INDEX IF EXISTS metrics_flow_id;
+DROP INDEX IF EXISTS metrics_execution_id;
+DROP INDEX IF EXISTS metrics_timestamp;
+
+ALTER TABLE metrics drop column "deleted";
\ No newline at end of file
diff --git a/jdbc-postgres/src/main/resources/migrations/postgres/V1_52__logs_metrics_deleted_indices.sql b/jdbc-postgres/src/main/resources/migrations/postgres/V1_52__logs_metrics_deleted_indices.sql
new file mode 100644
index 0000000000..20e53dbe8f
--- /dev/null
+++ b/jdbc-postgres/src/main/resources/migrations/postgres/V1_52__logs_metrics_deleted_indices.sql
@@ -0,0 +1,8 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id ON logs (execution_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__task_id ON logs (execution_id, task_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__taskrun_id ON logs (execution_id, taskrun_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_namespace_flow ON logs (tenant_id, timestamp, level, namespace, flow_id);
+
+CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_flow_id ON metrics (tenant_id, namespace, flow_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_execution_id ON metrics (execution_id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_timestamp ON metrics (tenant_id, timestamp);
\ No newline at end of file
diff --git a/jdbc-postgres/src/test/resources/application-test.yml b/jdbc-postgres/src/test/resources/application-test.yml
index c4413c2352..57f2c34639 100644
--- a/jdbc-postgres/src/test/resources/application-test.yml
+++ b/jdbc-postgres/src/test/resources/application-test.yml
@@ -15,6 +15,11 @@ flyway:
# We must ignore missing migrations as a V6 wrong migration was created and replaced by the V11
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
+ properties:
+ flyway:
+ postgresql:
+ transactional:
+ lock: false
kestra:
server-type: STANDALONE
diff --git a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcLogRepository.java b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcLogRepository.java
index 412dce467b..8bc17a4252 100644
--- a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcLogRepository.java
+++ b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcLogRepository.java
@@ -257,10 +257,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
- // The deleted field is not used, so ti will always be false.
- // We add it here to be sure to use the correct index.
- .where(field("deleted", Boolean.class).eq(false))
- .and(field("execution_id", String.class).eq(execution.getId()))
+ .where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -273,10 +270,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
- // The deleted field is not used, so ti will always be false.
- // We add it here to be sure to use the correct index.
- .where(field("deleted", Boolean.class).eq(false))
- .and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
+ .where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@@ -496,5 +490,15 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
});
}
+ @Override
+ protected Condition defaultFilter(String tenantId) {
+ return buildTenantCondition(tenantId);
+ }
+
+ @Override
+ protected Condition defaultFilter() {
+ return DSL.trueCondition();
+ }
+
abstract protected Field formatDateField(String dateField, DateUtils.GroupType groupType);
}
diff --git a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcMetricRepository.java b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcMetricRepository.java
index d39e9e07ad..55f1e81ec4 100644
--- a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcMetricRepository.java
+++ b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcMetricRepository.java
@@ -185,10 +185,7 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
- // The deleted field is not used, so ti will always be false.
- // We add it here to be sure to use the correct index.
- .where(field("deleted", Boolean.class).eq(false))
- .and(field("execution_id", String.class).eq(execution.getId()))
+ .where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -201,14 +198,21 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
- // The deleted field is not used, so ti will always be false.
- // We add it here to be sure to use the correct index.
- .where(field("deleted", Boolean.class).eq(false))
- .and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
+ .where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
+ @Override
+ protected Condition defaultFilter(String tenantId) {
+ return buildTenantCondition(tenantId);
+ }
+
+ @Override
+ protected Condition defaultFilter() {
+ return DSL.trueCondition();
+ }
+
private List queryDistinct(String tenantId, Condition condition, String field) {
return this.jdbcRepository
.getDslContextWrapper()