Compare commits

...

2 Commits

Author SHA1 Message Date
Loïc Mathieu
63e11c7d94 chore(system): remove unused deleted column in logs and metrics 2025-12-18 18:07:02 +01:00
Loïc Mathieu
11e199da33 chore(system): implements soft deletion consistenly accross entities 2025-12-18 18:07:02 +01:00
28 changed files with 181 additions and 86 deletions

View File

@@ -137,6 +137,11 @@ flyway:
# We must ignore missing migrations as we delete some wrong or not used anymore migrations
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
mysql:
enabled: true
locations:

View File

@@ -1,5 +0,0 @@
package io.kestra.core.models;
public interface DeletedInterface {
boolean isDeleted();
}

View File

@@ -0,0 +1,18 @@
package io.kestra.core.models;
/**
* This interface marks entities that implement a soft deletion mechanism.
* Soft deletion is based on a <code>deleted</code> field that is set to <code>true</code> when the entity is deleted.
* Physical deletion either never occurs or occurs in a dedicated purge mechanism.
*/
public interface SoftDeletable<T> {
/**
* Whether en entity is deleted or not.
*/
boolean isDeleted();
/**
* Delete the current entity: set its <code>deleted</code> field to <code>true</code>.
*/
T toDeleted();
}

View File

@@ -1,7 +1,7 @@
package io.kestra.core.models.dashboards;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.dashboards.charts.Chart;
import io.kestra.core.utils.IdUtils;
@@ -26,7 +26,7 @@ import java.util.Objects;
@NoArgsConstructor
@Introspected
@ToString
public class Dashboard implements HasUID, DeletedInterface {
public class Dashboard implements HasUID, SoftDeletable<Dashboard> {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
private String tenantId;
@@ -71,6 +71,7 @@ public class Dashboard implements HasUID, DeletedInterface {
);
}
@Override
public Dashboard toDeleted() {
return this.toBuilder()
.deleted(true)

View File

@@ -11,7 +11,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import io.kestra.core.debug.Breakpoint;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.Label;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.Flow;
@@ -53,7 +53,7 @@ import java.util.zip.CRC32;
@AllArgsConstructor
@ToString
@EqualsAndHashCode
public class Execution implements DeletedInterface, TenantInterface {
public class Execution implements SoftDeletable<Execution>, TenantInterface {
@With
@Hidden
@@ -1111,7 +1111,7 @@ public class Execution implements DeletedInterface, TenantInterface {
.toList();
}
@Override
public Execution toDeleted() {
return this.toBuilder()
.deleted(true)

View File

@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -22,7 +21,7 @@ import java.util.stream.Stream;
@Value
@Builder(toBuilder = true)
public class LogEntry implements DeletedInterface, TenantInterface {
public class LogEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -57,10 +56,6 @@ public class LogEntry implements DeletedInterface, TenantInterface {
String message;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

View File

@@ -1,7 +1,6 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.executions.metrics.Gauge;
@@ -18,7 +17,7 @@ import jakarta.validation.constraints.Pattern;
@Value
@Builder(toBuilder = true)
public class MetricEntry implements DeletedInterface, TenantInterface {
public class MetricEntry implements TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -54,10 +53,6 @@ public class MetricEntry implements DeletedInterface, TenantInterface {
@Nullable
Map<String, String> tags;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

View File

@@ -342,6 +342,7 @@ public class Flow extends AbstractFlow implements HasUID {
}
}
@Override
public Flow toDeleted() {
return this.toBuilder()
.revision(this.revision + 1)

View File

@@ -58,4 +58,9 @@ public class FlowForExecution extends AbstractFlow {
public String getSource() {
return null;
}
@Override
public FlowForExecution toDeleted() {
throw new UnsupportedOperationException("Can't delete a FlowForExecution");
}
}

View File

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.Label;
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
* The base interface for FLow.
*/
@JsonDeserialize(as = GenericFlow.class)
public interface FlowInterface extends FlowId, DeletedInterface, TenantInterface, HasUID, HasSource {
public interface FlowInterface extends FlowId, SoftDeletable<FlowInterface>, TenantInterface, HasUID, HasSource {
Pattern YAML_REVISION_MATCHER = Pattern.compile("(?m)^revision: \\d+\n?");

View File

@@ -96,4 +96,9 @@ public class GenericFlow extends AbstractFlow implements HasUID {
public List<GenericTrigger> getTriggers() {
return Optional.ofNullable(triggers).orElse(List.of());
}
@Override
public FlowInterface toDeleted() {
throw new UnsupportedOperationException("Can't delete a GenericFlow");
}
}

View File

@@ -1,6 +1,6 @@
package io.kestra.core.models.kv;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.kv.KVEntry;
@@ -22,7 +22,7 @@ import java.util.Optional;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
public class PersistedKvMetadata implements DeletedInterface, TenantInterface, HasUID {
public class PersistedKvMetadata implements SoftDeletable<PersistedKvMetadata>, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
@@ -83,6 +83,7 @@ public class PersistedKvMetadata implements DeletedInterface, TenantInterface, H
return this.toBuilder().updated(Instant.now()).last(true).build();
}
@Override
public PersistedKvMetadata toDeleted() {
return this.toBuilder().updated(Instant.now()).deleted(true).build();
}

View File

@@ -17,8 +17,4 @@ public class Namespace implements NamespaceInterface {
@NotNull
@Pattern(regexp="^[a-z0-9][a-z0-9._-]*")
protected String id;
@NotNull
@Builder.Default
boolean deleted = false;
}

View File

@@ -1,9 +1,8 @@
package io.kestra.core.models.namespaces;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
public interface NamespaceInterface extends DeletedInterface, HasUID {
public interface NamespaceInterface extends HasUID {
String getId();

View File

@@ -2,8 +2,8 @@ package io.kestra.core.models.namespaces.files;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.FileAttributes;
import io.kestra.core.storages.NamespaceFile;
@@ -24,7 +24,7 @@ import java.time.Instant;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
public class NamespaceFileMetadata implements DeletedInterface, TenantInterface, HasUID {
public class NamespaceFileMetadata implements SoftDeletable<NamespaceFileMetadata>, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
@@ -116,6 +116,7 @@ public class NamespaceFileMetadata implements DeletedInterface, TenantInterface,
return this.toBuilder().updated(saveDate).last(true).build();
}
@Override
public NamespaceFileMetadata toDeleted() {
return this.toBuilder().deleted(true).updated(Instant.now()).build();
}

View File

@@ -7,8 +7,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.validations.ManualConstraintViolation;
@@ -35,7 +35,7 @@ import jakarta.validation.constraints.Pattern;
@Introspected
@ToString
@EqualsAndHashCode
public class Template implements DeletedInterface, TenantInterface, HasUID {
public class Template implements SoftDeletable<Template>, TenantInterface, HasUID {
private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
@Override
@@ -141,6 +141,7 @@ public class Template implements DeletedInterface, TenantInterface, HasUID {
}
}
@Override
public Template toDeleted() {
return new Template(
this.tenantId,

View File

@@ -36,7 +36,7 @@ public interface KvMetadataRepositoryInterface extends SaveRepositoryInterface<P
);
default PersistedKvMetadata delete(PersistedKvMetadata persistedKvMetadata) throws IOException {
return this.save(persistedKvMetadata.toBuilder().deleted(true).build());
return this.save(persistedKvMetadata.toDeleted());
}
/**

View File

@@ -1,7 +1,7 @@
package io.kestra.core.test;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
@@ -25,7 +25,7 @@ import java.util.List;
@ToString
@EqualsAndHashCode
@TestSuiteValidation
public class TestSuite implements HasUID, TenantInterface, DeletedInterface, HasSource {
public class TestSuite implements HasUID, TenantInterface, SoftDeletable<TestSuite>, HasSource {
@NotNull
@NotBlank
@@ -85,10 +85,6 @@ public class TestSuite implements HasUID, TenantInterface, DeletedInterface, Has
);
}
public TestSuite delete() {
return this.toBuilder().deleted(true).build();
}
public TestSuite disable() {
var disabled = true;
return this.toBuilder()
@@ -120,4 +116,9 @@ public class TestSuite implements HasUID, TenantInterface, DeletedInterface, Has
return yamlSource + String.format("\ndisabled: %s", disabled);
}
@Override
public TestSuite toDeleted() {
return toBuilder().deleted(true).build();
}
}

View File

@@ -1,6 +1,6 @@
package io.kestra.core.test;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.test.flow.UnitTestResult;
@@ -24,7 +24,7 @@ public record TestSuiteRunEntity(
String flowId,
TestState state,
List<UnitTestResult> results
) implements DeletedInterface, TenantInterface, HasUID {
) implements SoftDeletable<TestSuiteRunEntity>, TenantInterface, HasUID {
public static TestSuiteRunEntity create(String tenantId, TestSuiteUid testSuiteUid, TestSuiteRunResult testSuiteRunResult) {
return new TestSuiteRunEntity(
@@ -43,23 +43,6 @@ public record TestSuiteRunEntity(
);
}
public TestSuiteRunEntity delete() {
return new TestSuiteRunEntity(
this.uid,
this.id,
this.tenantId,
true,
this.startDate,
this.endDate,
this.testSuiteId,
this.testSuiteUid,
this.namespace,
this.flowId,
this.state,
this.results
);
}
/**
* only used for backup
* @param newTenantId the tenant to migrate to
@@ -86,6 +69,24 @@ public record TestSuiteRunEntity(
return this.deleted;
}
@Override
public TestSuiteRunEntity toDeleted() {
return new TestSuiteRunEntity(
this.uid,
this.id,
this.tenantId,
true,
this.startDate,
this.endDate,
this.testSuiteId,
this.testSuiteUid,
this.namespace,
this.flowId,
this.state,
this.results
);
}
@Override
public String getTenantId() {
return this.tenantId;

View File

@@ -0,0 +1,22 @@
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
CREATE INDEX IF NOT EXISTS logs_execution_id ON logs ("execution_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__task_id ON logs ("execution_id", "task_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__taskrun_id ON logs ("execution_id", "taskrun_id");
CREATE INDEX IF NOT EXISTS logs_namespace_flow ON logs ("tenant_id", "timestamp", "level", "namespace", "flow_id");
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";
CREATE INDEX IF NOT EXISTS metrics_flow_id ON metrics ("tenant_id", "namespace", "flow_id");
CREATE INDEX IF NOT EXISTS metrics_execution_id ON metrics ("execution_id");
CREATE INDEX IF NOT EXISTS metrics_timestamp ON metrics ("tenant_id", "timestamp");

View File

@@ -0,0 +1,22 @@
ALTER TABLE logs DROP INDEX ix_execution_id;
ALTER TABLE logs DROP INDEX ix_execution_id__task_id;
ALTER TABLE logs DROP INDEX ix_execution_id__taskrun_id;
ALTER TABLE logs DROP INDEX ix_namespace_flow;
ALTER table logs drop column `deleted`;
ALTER TABLE logs ADD INDEX ix_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__task_id (`execution_id`, `task_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__taskrun_id (`execution_id`, `taskrun_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_namespace_flow (`tenant_id`, `timestamp`, `level`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics DROP INDEX metrics_flow_id;
ALTER TABLE metrics DROP INDEX ix_metrics_execution_id;
ALTER TABLE metrics DROP INDEX metrics_timestamp;
ALTER TABLE metrics drop column `deleted`;
ALTER TABLE metrics ADD INDEX ix_metrics_flow_id (`tenant_id`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_timestamp (`tenant_id`, `timestamp`), ALGORITHM=INPLACE, LOCK=NONE;

View File

@@ -0,0 +1,13 @@
-- Indices will be re-created by the next migration
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";

View File

@@ -0,0 +1,8 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id ON logs (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__task_id ON logs (execution_id, task_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__taskrun_id ON logs (execution_id, taskrun_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_namespace_flow ON logs (tenant_id, timestamp, level, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_flow_id ON metrics (tenant_id, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_execution_id ON metrics (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_timestamp ON metrics (tenant_id, timestamp);

View File

@@ -15,6 +15,11 @@ flyway:
# We must ignore missing migrations as a V6 wrong migration was created and replaced by the V11
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
kestra:
server-type: STANDALONE

View File

@@ -257,10 +257,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -273,10 +270,7 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@@ -496,5 +490,15 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
abstract protected Field<Date> formatDateField(String dateField, DateUtils.GroupType groupType);
}

View File

@@ -185,10 +185,7 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.where(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -201,14 +198,21 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
private List<String> queryDistinct(String tenantId, Condition condition, String field) {
return this.jdbcRepository
.getDslContextWrapper()

View File

@@ -57,7 +57,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -140,7 +140,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -195,7 +195,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -246,7 +246,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -330,7 +330,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -378,7 +378,6 @@ class DashboardControllerTest {
.namespace(fakeNamespace)
.level(Level.INFO)
.attemptNumber(1)
.deleted(false)
.executionId(fakeExecutionId)
.tenantId(MAIN_TENANT)
.executionKind(ExecutionKind.NORMAL)
@@ -447,7 +446,6 @@ class DashboardControllerTest {
.namespace(fakeNamespace)
.level(Level.INFO)
.attemptNumber(1)
.deleted(false)
.executionId(fakeExecutionId)
.tenantId(MAIN_TENANT)
.executionKind(ExecutionKind.NORMAL)

View File

@@ -57,7 +57,6 @@ public class NamespaceControllerTest {
);
assertThat(namespace.getId()).isEqualTo("my.ns");
assertThat(namespace.isDeleted()).isFalse();
}
@SuppressWarnings("unchecked")