jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(system): remove unused deleted column in logs and metrics

Browse Source
This commit is contained in:
Loïc Mathieu
2025-11-18 15:00:33 +01:00
parent 11e199da33
commit 63e11c7d94
18 changed files with 116 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cli/src/main/resources/application.yml
View File

@@ -137,6 +137,11 @@ flyway:
# We must ignore missing migrations as we delete some wrong or not used anymore migrations
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
mysql:
enabled: true
locations:

2
core/src/main/java/io/kestra/core/models/SoftDeletion.java → core/src/main/java/io/kestra/core/models/SoftDeletable.java
View File

@@ -5,7 +5,7 @@ package io.kestra.core.models;
* Soft deletion is based on a <code>deleted</code> field that is set to <code>true</code> when the entity is deleted.
* Physical deletion either never occurs or occurs in a dedicated purge mechanism.
*/
public interface SoftDeletion<T> {
public interface SoftDeletable<T> {
/**
* Whether en entity is deleted or not.
*/

4
core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java
View File

@@ -1,7 +1,7 @@
package io.kestra.core.models.dashboards;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.dashboards.charts.Chart;
import io.kestra.core.utils.IdUtils;
@@ -26,7 +26,7 @@ import java.util.Objects;
@NoArgsConstructor
@Introspected
@ToString
public class Dashboard implements HasUID, SoftDeletion<Dashboard> {
public class Dashboard implements HasUID, SoftDeletable<Dashboard> {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
private String tenantId;

4
core/src/main/java/io/kestra/core/models/executions/Execution.java
View File

@@ -11,7 +11,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import io.kestra.core.debug.Breakpoint;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.Label;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.Flow;
@@ -53,7 +53,7 @@ import java.util.zip.CRC32;
@AllArgsConstructor
@ToString
@EqualsAndHashCode
public class Execution implements SoftDeletion<Execution>, TenantInterface {
public class Execution implements SoftDeletable<Execution>, TenantInterface {
@With
@Hidden

7
core/src/main/java/io/kestra/core/models/executions/LogEntry.java
View File

@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -22,7 +21,7 @@ import java.util.stream.Stream;
@Value
@Builder(toBuilder = true)
public class LogEntry implements DeletedInterface, TenantInterface {
public class LogEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -57,10 +56,6 @@ public class LogEntry implements DeletedInterface, TenantInterface {
String message;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

7
core/src/main/java/io/kestra/core/models/executions/MetricEntry.java
View File

@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.executions.metrics.Gauge;
@@ -18,7 +17,7 @@ import jakarta.validation.constraints.Pattern;
@Value
@Builder(toBuilder = true)
public class MetricEntry implements DeletedInterface, TenantInterface {
public class MetricEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -54,10 +53,6 @@ public class MetricEntry implements DeletedInterface, TenantInterface {
@Nullable
Map<String, String> tags;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

4
core/src/main/java/io/kestra/core/models/flows/FlowInterface.java
View File

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.Label;
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
* The base interface for FLow.
*/
@JsonDeserialize(as = GenericFlow.class)
public interface FlowInterface extends FlowId, SoftDeletion<FlowInterface>, TenantInterface, HasUID, HasSource {
public interface FlowInterface extends FlowId, SoftDeletable<FlowInterface>, TenantInterface, HasUID, HasSource {
Pattern YAML_REVISION_MATCHER = Pattern.compile("(?m)^revision: \\d+\n?");

4
core/src/main/java/io/kestra/core/models/kv/PersistedKvMetadata.java
View File

@@ -1,6 +1,6 @@
package io.kestra.core.models.kv;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.kv.KVEntry;
@@ -22,7 +22,7 @@ import java.util.Optional;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
public class PersistedKvMetadata implements SoftDeletion<PersistedKvMetadata>, TenantInterface, HasUID {
public class PersistedKvMetadata implements SoftDeletable<PersistedKvMetadata>, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")

4
core/src/main/java/io/kestra/core/models/templates/Template.java
View File

@@ -7,8 +7,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.validations.ManualConstraintViolation;
@@ -35,7 +35,7 @@ import jakarta.validation.constraints.Pattern;
@Introspected
@ToString
@EqualsAndHashCode
public class Template implements SoftDeletion<Template>, TenantInterface, HasUID {
public class Template implements SoftDeletable<Template>, TenantInterface, HasUID {
private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
@Override

4
core/src/main/java/io/kestra/core/test/TestSuite.java
View File

@@ -1,7 +1,7 @@
package io.kestra.core.test;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
@@ -25,7 +25,7 @@ import java.util.List;
@ToString
@EqualsAndHashCode
@TestSuiteValidation
public class TestSuite implements HasUID, TenantInterface, SoftDeletion<TestSuite>, HasSource {
public class TestSuite implements HasUID, TenantInterface, SoftDeletable<TestSuite>, HasSource {
@NotNull
@NotBlank

4
core/src/main/java/io/kestra/core/test/TestSuiteRunEntity.java
View File

@@ -1,6 +1,6 @@
package io.kestra.core.test;
import io.kestra.core.models.SoftDeletion;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.test.flow.UnitTestResult;
@@ -24,7 +24,7 @@ public record TestSuiteRunEntity(
String flowId,
TestState state,
List<UnitTestResult> results
) implements SoftDeletion<TestSuiteRunEntity>, TenantInterface, HasUID {
) implements SoftDeletable<TestSuiteRunEntity>, TenantInterface, HasUID {
public static TestSuiteRunEntity create(String tenantId, TestSuiteUid testSuiteUid, TestSuiteRunResult testSuiteRunResult) {
return new TestSuiteRunEntity(

22
jdbc-h2/src/main/resources/migrations/h2/V1_51__logs_metrics_deleted.sql Normal file
View File

@@ -0,0 +1,22 @@
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
CREATE INDEX IF NOT EXISTS logs_execution_id ON logs ("execution_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__task_id ON logs ("execution_id", "task_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__taskrun_id ON logs ("execution_id", "taskrun_id");
CREATE INDEX IF NOT EXISTS logs_namespace_flow ON logs ("tenant_id", "timestamp", "level", "namespace", "flow_id");
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";
CREATE INDEX IF NOT EXISTS metrics_flow_id ON metrics ("tenant_id", "namespace", "flow_id");
CREATE INDEX IF NOT EXISTS metrics_execution_id ON metrics ("execution_id");
CREATE INDEX IF NOT EXISTS metrics_timestamp ON metrics ("tenant_id", "timestamp");

22
jdbc-mysql/src/main/resources/migrations/mysql/V1_52__logs_metrics_deleted.sql Normal file
View File

@@ -0,0 +1,22 @@
ALTER TABLE logs DROP INDEX ix_execution_id;
ALTER TABLE logs DROP INDEX ix_execution_id__task_id;
ALTER TABLE logs DROP INDEX ix_execution_id__taskrun_id;
ALTER TABLE logs DROP INDEX ix_namespace_flow;
ALTER table logs drop column `deleted`;
ALTER TABLE logs ADD INDEX ix_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__task_id (`execution_id`, `task_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__taskrun_id (`execution_id`, `taskrun_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_namespace_flow (`tenant_id`, `timestamp`, `level`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics DROP INDEX metrics_flow_id;
ALTER TABLE metrics DROP INDEX ix_metrics_execution_id;
ALTER TABLE metrics DROP INDEX metrics_timestamp;
ALTER TABLE metrics drop column `deleted`;
ALTER TABLE metrics ADD INDEX ix_metrics_flow_id (`tenant_id`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_timestamp (`tenant_id`, `timestamp`), ALGORITHM=INPLACE, LOCK=NONE;

13
jdbc-postgres/src/main/resources/migrations/postgres/V1_51__logs_metrics_deleted.sql Normal file
View File

@@ -0,0 +1,13 @@
-- Indices will be re-created by the next migration
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";

8
jdbc-postgres/src/main/resources/migrations/postgres/V1_52__logs_metrics_deleted_indices.sql Normal file
View File

@@ -0,0 +1,8 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id ON logs (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__task_id ON logs (execution_id, task_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__taskrun_id ON logs (execution_id, taskrun_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_namespace_flow ON logs (tenant_id, timestamp, level, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_flow_id ON metrics (tenant_id, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_execution_id ON metrics (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_timestamp ON metrics (tenant_id, timestamp);

5
jdbc-postgres/src/test/resources/application-test.yml
View File

@@ -15,6 +15,11 @@ flyway:
# We must ignore missing migrations as a V6 wrong migration was created and replaced by the V11
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
kestra:
server-type: STANDALONE

20
jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcLogRepository.java
View File

@@ -257,10 +257,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -273,10 +270,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@@ -496,5 +490,15 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
abstract protected Field<Date> formatDateField(String dateField, DateUtils.GroupType groupType);
}

20
jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcMetricRepository.java
View File

@@ -185,10 +185,7 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -201,14 +198,21 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
private List<String> queryDistinct(String tenantId, Condition condition, String field) {
return this.jdbcRepository
.getDslContextWrapper()