Compare commits

..

3 Commits

Author SHA1 Message Date
AJ Emerich
42c8334e2e Merge branch 'develop' into docs/purgeFiles 2025-12-18 04:02:10 -06:00
AJ Emerich
123d7fb426 Update core/src/main/java/io/kestra/plugin/core/namespace/PurgeFiles.java 2025-12-17 15:13:09 +01:00
AJ Emerich
e0c3cfa1f9 docs(PurgeFiles): update documentation and example 2025-12-17 12:26:57 +01:00
80 changed files with 340 additions and 726 deletions

View File

@@ -43,7 +43,7 @@ jobs:
# Upload dependency check report
- name: Upload dependency check report
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v5
if: ${{ always() }}
with:
name: dependency-check-report

View File

@@ -137,11 +137,6 @@ flyway:
# We must ignore missing migrations as we delete some wrong or not used anymore migrations
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
mysql:
enabled: true
locations:

View File

@@ -1,37 +0,0 @@
package io.kestra.core.exceptions;
import io.kestra.core.models.flows.Data;
import io.kestra.core.models.flows.Input;
import io.kestra.core.models.flows.Output;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Exception that can be thrown when Inputs/Outputs have validation problems.
*/
public class InputOutputValidationException extends KestraRuntimeException {
public InputOutputValidationException(String message) {
super(message);
}
public static InputOutputValidationException of( String message, Input<?> input){
String inputMessage = "Invalid value for input" + " `" + input.getId() + "`. Cause: " + message;
return new InputOutputValidationException(inputMessage);
}
public static InputOutputValidationException of( String message, Output output){
String outputMessage = "Invalid value for output" + " `" + output.getId() + "`. Cause: " + message;
return new InputOutputValidationException(outputMessage);
}
public static InputOutputValidationException of(String message){
return new InputOutputValidationException(message);
}
public static InputOutputValidationException merge(Set<InputOutputValidationException> exceptions){
String combinedMessage = exceptions.stream()
.map(InputOutputValidationException::getMessage)
.collect(Collectors.joining(System.lineSeparator()));
throw new InputOutputValidationException(combinedMessage);
}
}

View File

@@ -1,8 +1,6 @@
package io.kestra.core.exceptions;
import java.io.Serial;
import java.util.List;
import java.util.stream.Collectors;
/**
* The top-level {@link KestraRuntimeException} for non-recoverable errors.

View File

@@ -0,0 +1,5 @@
package io.kestra.core.models;
public interface DeletedInterface {
boolean isDeleted();
}

View File

@@ -1,18 +0,0 @@
package io.kestra.core.models;
/**
* This interface marks entities that implement a soft deletion mechanism.
* Soft deletion is based on a <code>deleted</code> field that is set to <code>true</code> when the entity is deleted.
* Physical deletion either never occurs or occurs in a dedicated purge mechanism.
*/
public interface SoftDeletable<T> {
/**
* Whether en entity is deleted or not.
*/
boolean isDeleted();
/**
* Delete the current entity: set its <code>deleted</code> field to <code>true</code>.
*/
T toDeleted();
}

View File

@@ -1,7 +1,7 @@
package io.kestra.core.models.dashboards;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.dashboards.charts.Chart;
import io.kestra.core.utils.IdUtils;
@@ -26,7 +26,7 @@ import java.util.Objects;
@NoArgsConstructor
@Introspected
@ToString
public class Dashboard implements HasUID, SoftDeletable<Dashboard> {
public class Dashboard implements HasUID, DeletedInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
private String tenantId;
@@ -71,7 +71,6 @@ public class Dashboard implements HasUID, SoftDeletable<Dashboard> {
);
}
@Override
public Dashboard toDeleted() {
return this.toBuilder()
.deleted(true)

View File

@@ -11,7 +11,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import io.kestra.core.debug.Breakpoint;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.Label;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.Flow;
@@ -53,7 +53,7 @@ import java.util.zip.CRC32;
@AllArgsConstructor
@ToString
@EqualsAndHashCode
public class Execution implements SoftDeletable<Execution>, TenantInterface {
public class Execution implements DeletedInterface, TenantInterface {
@With
@Hidden
@@ -1111,7 +1111,7 @@ public class Execution implements SoftDeletable<Execution>, TenantInterface {
.toList();
}
@Override
public Execution toDeleted() {
return this.toBuilder()
.deleted(true)

View File

@@ -1,6 +1,7 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -21,7 +22,7 @@ import java.util.stream.Stream;
@Value
@Builder(toBuilder = true)
public class LogEntry implements TenantInterface {
public class LogEntry implements DeletedInterface, TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -56,6 +57,10 @@ public class LogEntry implements TenantInterface {
String message;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

View File

@@ -1,6 +1,7 @@
package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.executions.metrics.Gauge;
@@ -17,7 +18,7 @@ import jakarta.validation.constraints.Pattern;
@Value
@Builder(toBuilder = true)
public class MetricEntry implements TenantInterface {
public class MetricEntry implements DeletedInterface, TenantInterface {
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
String tenantId;
@@ -53,6 +54,10 @@ public class MetricEntry implements TenantInterface {
@Nullable
Map<String, String> tags;
@NotNull
@Builder.Default
boolean deleted = false;
@Nullable
ExecutionKind executionKind;

View File

@@ -3,7 +3,9 @@ package io.kestra.core.models.executions;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.tasks.FlowableTask;
import io.kestra.core.models.tasks.ResolvedTask;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.tasks.retrys.AbstractRetry;
import io.kestra.core.utils.IdUtils;
import io.swagger.v3.oas.annotations.Hidden;
@@ -93,16 +95,8 @@ public class TaskRun implements TenantInterface {
this.forceExecution
);
}
public TaskRun withStateAndAttempt(State.Type state) {
List<TaskRunAttempt> newAttempts = new ArrayList<>(this.attempts != null ? this.attempts : List.of());
if (newAttempts.isEmpty()) {
newAttempts.add(TaskRunAttempt.builder().state(new State(state)).build());
} else {
TaskRunAttempt updatedLast = newAttempts.getLast().withState(state);
newAttempts.set(newAttempts.size() - 1, updatedLast);
}
public TaskRun replaceState(State newState) {
return new TaskRun(
this.tenantId,
this.id,
@@ -112,9 +106,9 @@ public class TaskRun implements TenantInterface {
this.taskId,
this.parentTaskRunId,
this.value,
newAttempts,
this.attempts,
this.outputs,
this.state.withState(state),
newState,
this.iteration,
this.dynamic,
this.forceExecution

View File

@@ -1,5 +1,7 @@
package io.kestra.core.models.flows;
import io.kestra.core.models.validations.ManualConstraintViolation;
import jakarta.validation.ConstraintViolationException;
/**
* Interface for defining an identifiable and typed data.
@@ -27,4 +29,16 @@ public interface Data {
*/
String getDisplayName();
@SuppressWarnings("unchecked")
default ConstraintViolationException toConstraintViolationException(String message, Object value) {
Class<Data> cls = (Class<Data>) this.getClass();
return ManualConstraintViolation.toConstraintViolationException(
"Invalid " + (this instanceof Output ? "output" : "input") + " for `" + getId() + "`, " + message + ", but received `" + value + "`",
this,
cls,
this.getId(),
value
);
}
}

View File

@@ -342,7 +342,6 @@ public class Flow extends AbstractFlow implements HasUID {
}
}
@Override
public Flow toDeleted() {
return this.toBuilder()
.revision(this.revision + 1)

View File

@@ -58,9 +58,4 @@ public class FlowForExecution extends AbstractFlow {
public String getSource() {
return null;
}
@Override
public FlowForExecution toDeleted() {
throw new UnsupportedOperationException("Can't delete a FlowForExecution");
}
}

View File

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.Label;
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
* The base interface for FLow.
*/
@JsonDeserialize(as = GenericFlow.class)
public interface FlowInterface extends FlowId, SoftDeletable<FlowInterface>, TenantInterface, HasUID, HasSource {
public interface FlowInterface extends FlowId, DeletedInterface, TenantInterface, HasUID, HasSource {
Pattern YAML_REVISION_MATCHER = Pattern.compile("(?m)^revision: \\d+\n?");

View File

@@ -96,9 +96,4 @@ public class GenericFlow extends AbstractFlow implements HasUID {
public List<GenericTrigger> getTriggers() {
return Optional.ofNullable(triggers).orElse(List.of());
}
@Override
public FlowInterface toDeleted() {
throw new UnsupportedOperationException("Can't delete a GenericFlow");
}
}

View File

@@ -1,12 +1,10 @@
package io.kestra.core.models.flows.input;
import io.kestra.core.exceptions.InputOutputValidationException;
import io.kestra.core.models.flows.Input;
import jakarta.annotation.Nullable;
import jakarta.validation.ConstraintViolationException;
import jakarta.validation.constraints.NotNull;
import java.util.Set;
/**
* Represents an input along with its associated value and validation state.
*
@@ -14,15 +12,15 @@ import java.util.Set;
* @param value The provided value for the input.
* @param enabled {@code true} if the input is enabled; {@code false} otherwise.
* @param isDefault {@code true} if the provided value is the default; {@code false} otherwise.
* @param exceptions The validation exceptions, if the input value is invalid; {@code null} otherwise.
* @param exception The validation exception, if the input value is invalid; {@code null} otherwise.
*/
public record InputAndValue(
Input<?> input,
Object value,
boolean enabled,
boolean isDefault,
Set<InputOutputValidationException> exceptions) {
ConstraintViolationException exception) {
/**
* Creates a new {@link InputAndValue} instance.
*

View File

@@ -7,7 +7,6 @@ import io.kestra.core.models.property.Property;
import io.kestra.core.models.validations.ManualConstraintViolation;
import io.kestra.core.validations.Regex;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.ConstraintViolation;
import jakarta.validation.ConstraintViolationException;
import jakarta.validation.constraints.NotNull;
import lombok.Builder;
@@ -15,7 +14,10 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.*;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
@SuperBuilder
@@ -75,35 +77,30 @@ public class MultiselectInput extends Input<List<String>> implements ItemTypeInt
@Override
public void validate(List<String> inputs) throws ConstraintViolationException {
Set<ConstraintViolation<?>> violations = new HashSet<>();
if (values != null && options != null) {
violations.add( ManualConstraintViolation.of(
throw ManualConstraintViolation.toConstraintViolationException(
"you can't define both `values` and `options`",
this,
MultiselectInput.class,
getId(),
""
));
);
}
if (!this.getAllowCustomValue()) {
for (String input : inputs) {
List<@Regex String> finalValues = this.values != null ? this.values : this.options;
if (!finalValues.contains(input)) {
violations.add(ManualConstraintViolation.of(
"value `" + input + "` doesn't match the values `" + finalValues + "`",
throw ManualConstraintViolation.toConstraintViolationException(
"it must match the values `" + finalValues + "`",
this,
MultiselectInput.class,
getId(),
input
));
);
}
}
}
if (!violations.isEmpty()) {
throw ManualConstraintViolation.toConstraintViolationException(violations);
}
}
/** {@inheritDoc} **/
@@ -148,7 +145,7 @@ public class MultiselectInput extends Input<List<String>> implements ItemTypeInt
String type = Optional.ofNullable(result).map(Object::getClass).map(Class::getSimpleName).orElse("<null>");
throw ManualConstraintViolation.toConstraintViolationException(
"Invalid expression result. Expected a list of strings",
"Invalid expression result. Expected a list of strings, but received " + type,
this,
MultiselectInput.class,
getId(),

View File

@@ -125,7 +125,7 @@ public class SelectInput extends Input<String> implements RenderableInput {
String type = Optional.ofNullable(result).map(Object::getClass).map(Class::getSimpleName).orElse("<null>");
throw ManualConstraintViolation.toConstraintViolationException(
"Invalid expression result. Expected a list of strings",
"Invalid expression result. Expected a list of strings, but received " + type,
this,
SelectInput.class,
getId(),

View File

@@ -1,6 +1,6 @@
package io.kestra.core.models.kv;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.kv.KVEntry;
@@ -22,7 +22,7 @@ import java.util.Optional;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
public class PersistedKvMetadata implements SoftDeletable<PersistedKvMetadata>, TenantInterface, HasUID {
public class PersistedKvMetadata implements DeletedInterface, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
@@ -83,7 +83,6 @@ public class PersistedKvMetadata implements SoftDeletable<PersistedKvMetadata>,
return this.toBuilder().updated(Instant.now()).last(true).build();
}
@Override
public PersistedKvMetadata toDeleted() {
return this.toBuilder().updated(Instant.now()).deleted(true).build();
}

View File

@@ -17,4 +17,8 @@ public class Namespace implements NamespaceInterface {
@NotNull
@Pattern(regexp="^[a-z0-9][a-z0-9._-]*")
protected String id;
@NotNull
@Builder.Default
boolean deleted = false;
}

View File

@@ -1,8 +1,9 @@
package io.kestra.core.models.namespaces;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
public interface NamespaceInterface extends HasUID {
public interface NamespaceInterface extends DeletedInterface, HasUID {
String getId();

View File

@@ -2,8 +2,8 @@ package io.kestra.core.models.namespaces.files;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.storages.FileAttributes;
import io.kestra.core.storages.NamespaceFile;
@@ -24,7 +24,7 @@ import java.time.Instant;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@ToString
@EqualsAndHashCode
public class NamespaceFileMetadata implements SoftDeletable<NamespaceFileMetadata>, TenantInterface, HasUID {
public class NamespaceFileMetadata implements DeletedInterface, TenantInterface, HasUID {
@With
@Hidden
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
@@ -116,7 +116,6 @@ public class NamespaceFileMetadata implements SoftDeletable<NamespaceFileMetadat
return this.toBuilder().updated(saveDate).last(true).build();
}
@Override
public NamespaceFileMetadata toDeleted() {
return this.toBuilder().deleted(true).updated(Instant.now()).build();
}

View File

@@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.google.common.annotations.VisibleForTesting;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextProperty;
import io.kestra.core.serializers.JacksonMapper;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
@@ -157,9 +156,9 @@ public class Property<T> {
/**
* Render a property, then convert it to its target type.<br>
* <p>
* This method is designed to be used only by the {@link RunContextProperty}.
* This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
*
* @see RunContextProperty#as(Class)
* @see io.kestra.core.runners.RunContextProperty#as(Class)
*/
public static <T> T as(Property<T> property, PropertyContext context, Class<T> clazz) throws IllegalVariableEvaluationException {
return as(property, context, clazz, Map.of());
@@ -168,57 +167,25 @@ public class Property<T> {
/**
* Render a property with additional variables, then convert it to its target type.<br>
* <p>
* This method is designed to be used only by the {@link RunContextProperty}.
* This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
*
* @see RunContextProperty#as(Class, Map)
* @see io.kestra.core.runners.RunContextProperty#as(Class, Map)
*/
public static <T> T as(Property<T> property, PropertyContext context, Class<T> clazz, Map<String, Object> variables) throws IllegalVariableEvaluationException {
if (property.skipCache || property.value == null) {
String rendered = context.render(property.expression, variables);
property.value = deserialize(rendered, clazz);
property.value = MAPPER.convertValue(rendered, clazz);
}
return property.value;
}
private static <T> T deserialize(Object rendered, Class<T> clazz) throws IllegalVariableEvaluationException {
try {
return MAPPER.convertValue(rendered, clazz);
} catch (IllegalArgumentException e) {
if (rendered instanceof String str) {
try {
return MAPPER.readValue(str, clazz);
} catch (JsonProcessingException ex) {
throw new IllegalVariableEvaluationException(ex);
}
}
throw new IllegalVariableEvaluationException(e);
}
}
private static <T> T deserialize(Object rendered, JavaType type) throws IllegalVariableEvaluationException {
try {
return MAPPER.convertValue(rendered, type);
} catch (IllegalArgumentException e) {
if (rendered instanceof String str) {
try {
return MAPPER.readValue(str, type);
} catch (JsonProcessingException ex) {
throw new IllegalVariableEvaluationException(ex);
}
}
throw new IllegalVariableEvaluationException(e);
}
}
/**
* Render a property then convert it as a list of target type.<br>
* <p>
* This method is designed to be used only by the {@link RunContextProperty}.
* This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
*
* @see RunContextProperty#asList(Class)
* @see io.kestra.core.runners.RunContextProperty#asList(Class)
*/
public static <T, I> T asList(Property<T> property, PropertyContext context, Class<I> itemClazz) throws IllegalVariableEvaluationException {
return asList(property, context, itemClazz, Map.of());
@@ -227,39 +194,37 @@ public class Property<T> {
/**
* Render a property with additional variables, then convert it as a list of target type.<br>
* <p>
* This method is designed to be used only by the {@link RunContextProperty}.
* This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
*
* @see RunContextProperty#asList(Class, Map)
* @see io.kestra.core.runners.RunContextProperty#asList(Class, Map)
*/
@SuppressWarnings("unchecked")
public static <T, I> T asList(Property<T> property, PropertyContext context, Class<I> itemClazz, Map<String, Object> variables) throws IllegalVariableEvaluationException {
if (property.skipCache || property.value == null) {
JavaType type = MAPPER.getTypeFactory().constructCollectionLikeType(List.class, itemClazz);
String trimmedExpression = property.expression.trim();
// We need to detect if the expression is already a list or if it's a pebble expression (for eg. referencing a variable containing a list).
// Doing that allows us to, if it's an expression, first render then read it as a list.
if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
property.value = deserialize(context.render(property.expression, variables), type);
}
// Otherwise, if it's already a list, we read it as a list first then render it from run context which handle list rendering by rendering each item of the list
else {
List<?> asRawList = deserialize(property.expression, List.class);
property.value = (T) asRawList.stream()
.map(throwFunction(item -> {
Object rendered = null;
if (item instanceof String str) {
rendered = context.render(str, variables);
} else if (item instanceof Map map) {
rendered = context.render(map, variables);
}
if (rendered != null) {
return deserialize(rendered, itemClazz);
}
return item;
}))
.toList();
try {
String trimmedExpression = property.expression.trim();
// We need to detect if the expression is already a list or if it's a pebble expression (for eg. referencing a variable containing a list).
// Doing that allows us to, if it's an expression, first render then read it as a list.
if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
property.value = MAPPER.readValue(context.render(property.expression, variables), type);
}
// Otherwise, if it's already a list, we read it as a list first then render it from run context which handle list rendering by rendering each item of the list
else {
List<?> asRawList = MAPPER.readValue(property.expression, List.class);
property.value = (T) asRawList.stream()
.map(throwFunction(item -> {
if (item instanceof String str) {
return MAPPER.convertValue(context.render(str, variables), itemClazz);
} else if (item instanceof Map map) {
return MAPPER.convertValue(context.render(map, variables), itemClazz);
}
return item;
}))
.toList();
}
} catch (JsonProcessingException e) {
throw new IllegalVariableEvaluationException(e);
}
}
@@ -269,9 +234,9 @@ public class Property<T> {
/**
* Render a property then convert it as a map of target types.<br>
* <p>
* This method is designed to be used only by the {@link RunContextProperty}.
* This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
*
* @see RunContextProperty#asMap(Class, Class)
* @see io.kestra.core.runners.RunContextProperty#asMap(Class, Class)
*/
public static <T, K, V> T asMap(Property<T> property, RunContext runContext, Class<K> keyClass, Class<V> valueClass) throws IllegalVariableEvaluationException {
return asMap(property, runContext, keyClass, valueClass, Map.of());
@@ -283,7 +248,7 @@ public class Property<T> {
* This method is safe to be used as many times as you want as the rendering and conversion will be cached.
* Warning, due to the caching mechanism, this method is not thread-safe.
*
* @see RunContextProperty#asMap(Class, Class, Map)
* @see io.kestra.core.runners.RunContextProperty#asMap(Class, Class, Map)
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public static <T, K, V> T asMap(Property<T> property, RunContext runContext, Class<K> keyClass, Class<V> valueClass, Map<String, Object> variables) throws IllegalVariableEvaluationException {
@@ -295,12 +260,12 @@ public class Property<T> {
// We need to detect if the expression is already a map or if it's a pebble expression (for eg. referencing a variable containing a map).
// Doing that allows us to, if it's an expression, first render then read it as a map.
if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
property.value = deserialize(runContext.render(property.expression, variables), targetMapType);
property.value = MAPPER.readValue(runContext.render(property.expression, variables), targetMapType);
}
// Otherwise if it's already a map we read it as a map first then render it from run context which handle map rendering by rendering each entry of the map (otherwise it will fail with nested expressions in values for eg.)
else {
Map asRawMap = MAPPER.readValue(property.expression, Map.class);
property.value = deserialize(runContext.render(asRawMap, variables), targetMapType);
property.value = MAPPER.convertValue(runContext.render(asRawMap, variables), targetMapType);
}
} catch (JsonProcessingException e) {
throw new IllegalVariableEvaluationException(e);

View File

@@ -7,8 +7,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.validations.ManualConstraintViolation;
@@ -35,7 +35,7 @@ import jakarta.validation.constraints.Pattern;
@Introspected
@ToString
@EqualsAndHashCode
public class Template implements SoftDeletable<Template>, TenantInterface, HasUID {
public class Template implements DeletedInterface, TenantInterface, HasUID {
private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
@Override
@@ -141,7 +141,6 @@ public class Template implements SoftDeletable<Template>, TenantInterface, HasUI
}
}
@Override
public Template toDeleted() {
return new Template(
this.tenantId,

View File

@@ -67,11 +67,6 @@ public class ManualConstraintViolation<T> implements ConstraintViolation<T> {
invalidValue
)));
}
public static <T> ConstraintViolationException toConstraintViolationException(
Set<? extends ConstraintViolation<?>> constraintViolations
) {
return new ConstraintViolationException(constraintViolations);
}
public String getMessageTemplate() {
return "{messageTemplate}";

View File

@@ -36,7 +36,7 @@ public interface KvMetadataRepositoryInterface extends SaveRepositoryInterface<P
);
default PersistedKvMetadata delete(PersistedKvMetadata persistedKvMetadata) throws IOException {
return this.save(persistedKvMetadata.toDeleted());
return this.save(persistedKvMetadata.toBuilder().deleted(true).build());
}
/**

View File

@@ -3,8 +3,6 @@ package io.kestra.core.runners;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.kestra.core.encryption.EncryptionService;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.exceptions.InputOutputValidationException;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Data;
import io.kestra.core.models.flows.DependsOn;
@@ -19,6 +17,7 @@ import io.kestra.core.models.property.Property;
import io.kestra.core.models.property.PropertyContext;
import io.kestra.core.models.property.URIFetcher;
import io.kestra.core.models.tasks.common.EncryptedString;
import io.kestra.core.models.validations.ManualConstraintViolation;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.storages.StorageContext;
import io.kestra.core.storages.StorageInterface;
@@ -208,8 +207,8 @@ public class FlowInputOutput {
.filter(InputAndValue::enabled)
.map(it -> {
//TODO check to return all exception at-once.
if (it.exceptions() != null && !it.exceptions().isEmpty()) {
throw InputOutputValidationException.merge(it.exceptions());
if (it.exception() != null) {
throw it.exception();
}
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
})
@@ -293,9 +292,13 @@ public class FlowInputOutput {
try {
isInputEnabled = Boolean.TRUE.equals(runContext.renderTyped(dependsOnCondition.get()));
} catch (IllegalVariableEvaluationException e) {
resolvable.resolveWithError(
InputOutputValidationException.of("Invalid condition: " + e.getMessage())
);
resolvable.resolveWithError(ManualConstraintViolation.toConstraintViolationException(
"Invalid condition: " + e.getMessage(),
input,
(Class<Input>)input.getClass(),
input.getId(),
this
));
isInputEnabled = false;
}
}
@@ -328,7 +331,7 @@ public class FlowInputOutput {
// validate and parse input value
if (value == null) {
if (input.getRequired()) {
resolvable.resolveWithError(InputOutputValidationException.of("Missing required input:" + input.getId()));
resolvable.resolveWithError(input.toConstraintViolationException("missing required input", null));
} else {
resolvable.resolveWithValue(null);
}
@@ -338,18 +341,17 @@ public class FlowInputOutput {
parsedInput.ifPresent(parsed -> ((Input) resolvable.get().input()).validate(parsed.getValue()));
parsedInput.ifPresent(typed -> resolvable.resolveWithValue(typed.getValue()));
} catch (ConstraintViolationException e) {
Input<?> finalInput = input;
Set<InputOutputValidationException> exceptions = e.getConstraintViolations().stream()
.map(c-> InputOutputValidationException.of(c.getMessage(), finalInput))
.collect(Collectors.toSet());
resolvable.resolveWithError(exceptions);
ConstraintViolationException exception = e.getConstraintViolations().size() == 1 ?
input.toConstraintViolationException(List.copyOf(e.getConstraintViolations()).getFirst().getMessage(), value) :
input.toConstraintViolationException(e.getMessage(), value);
resolvable.resolveWithError(exception);
}
}
} catch (IllegalArgumentException e){
resolvable.resolveWithError(InputOutputValidationException.of(e.getMessage(), input));
}
catch (Exception e) {
resolvable.resolveWithError(InputOutputValidationException.of(e.getMessage()));
} catch (ConstraintViolationException e) {
resolvable.resolveWithError(e);
} catch (Exception e) {
ConstraintViolationException exception = input.toConstraintViolationException(e instanceof IllegalArgumentException ? e.getMessage() : e.toString(), resolvable.get().value());
resolvable.resolveWithError(exception);
}
return resolvable.get();
@@ -437,12 +439,8 @@ public class FlowInputOutput {
}
return entry;
});
}
catch (IllegalArgumentException e){
throw InputOutputValidationException.of(e.getMessage(), output);
}
catch (Exception e) {
throw InputOutputValidationException.of(e.getMessage());
} catch (Exception e) {
throw output.toConstraintViolationException(e.getMessage(), current);
}
})
.filter(Optional::isPresent)
@@ -505,7 +503,7 @@ public class FlowInputOutput {
if (matcher.matches()) {
yield current.toString();
} else {
throw new IllegalArgumentException("Invalid URI format.");
throw new IllegalArgumentException("Expected `URI` but received `" + current + "`");
}
}
case ARRAY, MULTISELECT -> {
@@ -535,7 +533,7 @@ public class FlowInputOutput {
} catch (IllegalArgumentException e) {
throw e;
} catch (Throwable e) {
throw new Exception(" errors:\n```\n" + e.getMessage() + "\n```");
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
}
}
@@ -567,30 +565,27 @@ public class FlowInputOutput {
}
public void isDefault(boolean isDefault) {
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exceptions());
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
}
public void setInput(final Input<?> input) {
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exceptions());
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
}
public void resolveWithEnabled(boolean enabled) {
this.input = new InputAndValue(this.input.input(), input.value(), enabled, this.input.isDefault(), this.input.exceptions());
this.input = new InputAndValue(this.input.input(), input.value(), enabled, this.input.isDefault(), this.input.exception());
markAsResolved();
}
public void resolveWithValue(@Nullable Object value) {
this.input = new InputAndValue(this.input.input(), value, this.input.enabled(), this.input.isDefault(), this.input.exceptions());
this.input = new InputAndValue(this.input.input(), value, this.input.enabled(), this.input.isDefault(), this.input.exception());
markAsResolved();
}
public void resolveWithError(@Nullable Set<InputOutputValidationException> exception) {
public void resolveWithError(@Nullable ConstraintViolationException exception) {
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), this.input.isDefault(), exception);
markAsResolved();
}
private void resolveWithError(@Nullable InputOutputValidationException exception){
resolveWithError(Collections.singleton(exception));
}
private void markAsResolved() {
this.isResolved = true;

View File

@@ -8,7 +8,6 @@ import io.micronaut.core.annotation.Nullable;
import io.pebbletemplates.pebble.PebbleEngine;
import io.pebbletemplates.pebble.extension.Extension;
import io.pebbletemplates.pebble.extension.Function;
import io.pebbletemplates.pebble.lexer.Syntax;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
@@ -38,13 +37,6 @@ public class PebbleEngineFactory {
return builder.build();
}
public PebbleEngine createWithCustomSyntax(Syntax syntax, Class<? extends Extension> extension) {
PebbleEngine.Builder builder = newPebbleEngineBuilder()
.syntax(syntax);
this.applicationContext.getBeansOfType(extension).forEach(builder::extension);
return builder.build();
}
public PebbleEngine createWithMaskedFunctions(VariableRenderer renderer, final List<String> functionsToMask) {
PebbleEngine.Builder builder = newPebbleEngineBuilder();

View File

@@ -35,10 +35,6 @@ public final class YamlParser {
return read(input, cls, type(cls));
}
public static <T> T parse(String input, Class<T> cls, Boolean strict) {
return strict ? read(input, cls, type(cls)) : readNonStrict(input, cls, type(cls));
}
public static <T> T parse(Map<String, Object> input, Class<T> cls, Boolean strict) {
ObjectMapper currentMapper = strict ? STRICT_MAPPER : NON_STRICT_MAPPER;
@@ -85,13 +81,6 @@ public final class YamlParser {
throw toConstraintViolationException(input, resource, e);
}
}
private static <T> T readNonStrict(String input, Class<T> objectClass, String resource) {
try {
return NON_STRICT_MAPPER.readValue(input, objectClass);
} catch (JsonProcessingException e) {
throw toConstraintViolationException(input, resource, e);
}
}
private static String formatYamlErrorMessage(String originalMessage, JsonProcessingException e) {
StringBuilder friendlyMessage = new StringBuilder();
if (originalMessage.contains("Expected a field name")) {

View File

@@ -754,7 +754,7 @@ public class ExecutionService {
var parentTaskRun = execution.findTaskRunByTaskRunId(taskRun.getParentTaskRunId());
Execution newExecution = execution;
if (parentTaskRun.getState().getCurrent() != State.Type.KILLED) {
newExecution = newExecution.withTaskRun(parentTaskRun.withStateAndAttempt(State.Type.KILLED));
newExecution = newExecution.withTaskRun(parentTaskRun.withState(State.Type.KILLED));
}
if (parentTaskRun.getParentTaskRunId() != null) {
return killParentTaskruns(parentTaskRun, newExecution);

View File

@@ -1,7 +1,7 @@
package io.kestra.core.test;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasSource;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
@@ -25,7 +25,7 @@ import java.util.List;
@ToString
@EqualsAndHashCode
@TestSuiteValidation
public class TestSuite implements HasUID, TenantInterface, SoftDeletable<TestSuite>, HasSource {
public class TestSuite implements HasUID, TenantInterface, DeletedInterface, HasSource {
@NotNull
@NotBlank
@@ -85,6 +85,10 @@ public class TestSuite implements HasUID, TenantInterface, SoftDeletable<TestSui
);
}
public TestSuite delete() {
return this.toBuilder().deleted(true).build();
}
public TestSuite disable() {
var disabled = true;
return this.toBuilder()
@@ -116,9 +120,4 @@ public class TestSuite implements HasUID, TenantInterface, SoftDeletable<TestSui
return yamlSource + String.format("\ndisabled: %s", disabled);
}
@Override
public TestSuite toDeleted() {
return toBuilder().deleted(true).build();
}
}

View File

@@ -1,6 +1,6 @@
package io.kestra.core.test;
import io.kestra.core.models.SoftDeletable;
import io.kestra.core.models.DeletedInterface;
import io.kestra.core.models.HasUID;
import io.kestra.core.models.TenantInterface;
import io.kestra.core.test.flow.UnitTestResult;
@@ -24,7 +24,7 @@ public record TestSuiteRunEntity(
String flowId,
TestState state,
List<UnitTestResult> results
) implements SoftDeletable<TestSuiteRunEntity>, TenantInterface, HasUID {
) implements DeletedInterface, TenantInterface, HasUID {
public static TestSuiteRunEntity create(String tenantId, TestSuiteUid testSuiteUid, TestSuiteRunResult testSuiteRunResult) {
return new TestSuiteRunEntity(
@@ -43,6 +43,23 @@ public record TestSuiteRunEntity(
);
}
public TestSuiteRunEntity delete() {
return new TestSuiteRunEntity(
this.uid,
this.id,
this.tenantId,
true,
this.startDate,
this.endDate,
this.testSuiteId,
this.testSuiteUid,
this.namespace,
this.flowId,
this.state,
this.results
);
}
/**
* only used for backup
* @param newTenantId the tenant to migrate to
@@ -69,24 +86,6 @@ public record TestSuiteRunEntity(
return this.deleted;
}
@Override
public TestSuiteRunEntity toDeleted() {
return new TestSuiteRunEntity(
this.uid,
this.id,
this.tenantId,
true,
this.startDate,
this.endDate,
this.testSuiteId,
this.testSuiteUid,
this.namespace,
this.flowId,
this.state,
this.results
);
}
@Override
public String getTenantId() {
return this.tenantId;

View File

@@ -2,8 +2,10 @@ package io.kestra.plugin.core.namespace;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import io.kestra.core.repositories.NamespaceFileMetadataRepositoryInterface;
import io.kestra.core.storages.Namespace;
import io.kestra.core.storages.NamespaceFile;
import io.kestra.plugin.core.kv.Version;
import io.micronaut.core.annotation.Introspected;
import lombok.Getter;
import lombok.NoArgsConstructor;

View File

@@ -26,25 +26,28 @@ import java.util.concurrent.atomic.AtomicLong;
@Getter
@NoArgsConstructor
@Schema(
title = "Delete expired keys globally for a specific namespace.",
description = "This task will delete expired keys from the Kestra KV store. By default, it will only delete expired keys, but you can choose to delete all keys by setting `expiredOnly` to false. You can also filter keys by a specific pattern and choose to include child namespaces."
title = "Purge namespace files for one or multiple namespaces.",
description = "This task purges namespace files (and their versions) stored in Kestra. You can restrict the purge to specific namespaces (or a namespace glob pattern), optionally include child namespaces, and filter files by a glob pattern. The purge strategy is controlled via `behavior` (e.g. keep the last N versions and/or delete versions older than a given date)."
)
@Plugin(
examples = {
@Example(
title = "Delete expired keys globally for a specific namespace, with or without including child namespaces.",
title = "Purge old versions of namespace files for a namespace tree.",
full = true,
code = """
id: purge_kv_store
id: purge_namespace_files
namespace: system
tasks:
- id: purge_kv
type: io.kestra.plugin.core.kv.PurgeKV
expiredOnly: true
- id: purge_files
type: io.kestra.plugin.core.namespace.PurgeFiles
namespaces:
- company
includeChildNamespaces: true
filePattern: "**/*.sql"
behavior:
type: version
before: "2025-01-01T00:00:00Z"
"""
)
}
@@ -116,7 +119,7 @@ public class PurgeFiles extends Task implements PurgeTask<NamespaceFile>, Runnab
@Getter
public static class Output implements io.kestra.core.models.tasks.Output {
@Schema(
title = "The number of purged KV pairs"
title = "The number of purged namespace file versions"
)
private Long size;
}

View File

@@ -1,36 +1,24 @@
package io.kestra.core.models.property;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.kestra.core.context.TestRunContextFactory;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.serializers.FileSerde;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.storages.Namespace;
import io.kestra.core.storages.NamespaceFile;
import io.kestra.core.storages.StorageInterface;
import io.kestra.plugin.core.namespace.Version;
import io.micronaut.core.annotation.Introspected;
import jakarta.inject.Inject;
import jakarta.validation.ConstraintViolationException;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.junit.jupiter.api.Test;
import org.slf4j.event.Level;
import reactor.core.publisher.Flux;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static java.util.Map.entry;
@@ -374,43 +362,10 @@ class PropertyTest {
assertThat(output.getMessages().getFirst().getValue()).isEqualTo("value1");
}
@Test
void jsonSubtype() throws JsonProcessingException, IllegalVariableEvaluationException {
Optional<WithSubtype> rendered = runContextFactory.of().render(
Property.<WithSubtype>ofExpression(JacksonMapper.ofJson().writeValueAsString(new MySubtype()))
).as(WithSubtype.class);
assertThat(rendered).isPresent();
assertThat(rendered.get()).isInstanceOf(MySubtype.class);
List<WithSubtype> renderedList = runContextFactory.of().render(
Property.<List<WithSubtype>>ofExpression(JacksonMapper.ofJson().writeValueAsString(List.of(new MySubtype())))
).asList(WithSubtype.class);
assertThat(renderedList).hasSize(1);
assertThat(renderedList.get(0)).isInstanceOf(MySubtype.class);
}
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY)
@JsonSubTypes({
@JsonSubTypes.Type(value = MySubtype.class, name = "mySubtype")
})
@Getter
@NoArgsConstructor
@Introspected
public abstract static class WithSubtype {
abstract public String getType();
}
@Getter
public static class MySubtype extends WithSubtype {
private final String type = "mySubtype";
}
@Builder
@Getter
private static class TestObj {
private String key;
private String value;
}
}
}

View File

@@ -78,7 +78,6 @@ public abstract class AbstractRunnerConcurrencyTest {
}
@Test
@FlakyTest(description = "Only flaky in CI")
@LoadFlows(value = {"flows/valids/flow-concurrency-queue-killed.yml"}, tenantId = "flow-concurrency-killed")
void flowConcurrencyKilled() throws Exception {
flowConcurrencyCaseTest.flowConcurrencyKilled("flow-concurrency-killed");

View File

@@ -6,7 +6,6 @@ import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.Label;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.flows.GenericFlow;
@@ -467,20 +466,4 @@ class ExecutionServiceTest {
assertThat(restart.getTaskRunList()).hasSize(2);
assertThat(restart.findTaskRunsByTaskId("make_error").getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
@Test
@LoadFlows({"flows/valids/each-pause.yaml"})
void killExecutionWithFlowableTask() throws Exception {
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "each-pause");
TaskRun childTaskRun = execution.getTaskRunList().stream().filter(tr -> tr.getTaskId().equals("pause")).toList().getFirst();
Execution killed = executionService.killParentTaskruns(childTaskRun,execution);
TaskRun parentTaskRun = killed.getTaskRunList().stream().filter(tr -> tr.getTaskId().equals("each_task")).toList().getFirst();
assertThat(parentTaskRun.getState().getCurrent()).isEqualTo(State.Type.KILLED);
assertThat(parentTaskRun.getAttempts().getLast().getState().getCurrent()).isEqualTo(State.Type.KILLED);
}
}

View File

@@ -239,7 +239,7 @@ class FlowInputOutputTest {
// Then
Assertions.assertEquals(2, values.size());
Assertions.assertFalse(values.get(1).enabled());
Assertions.assertNotNull(values.get(1).exceptions());
Assertions.assertNotNull(values.get(1).exception());
}
@Test
@@ -257,7 +257,7 @@ class FlowInputOutputTest {
List<InputAndValue> values = flowInputOutput.validateExecutionInputs(List.of(input), null, DEFAULT_TEST_EXECUTION, data).block();
// Then
Assertions.assertNull(values.getFirst().exceptions());
Assertions.assertNull(values.getFirst().exception());
Assertions.assertFalse(storageInterface.exists(MAIN_TENANT, null, URI.create(values.getFirst().value().toString())));
}

View File

@@ -2,7 +2,6 @@ package io.kestra.core.runners;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.CharStreams;
import io.kestra.core.exceptions.InputOutputValidationException;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.executions.Execution;
@@ -138,8 +137,8 @@ public class InputsTest {
void missingRequired() {
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
inputs.put("string", null);
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(inputs, MAIN_TENANT));
assertThat(e.getMessage()).contains("Missing required input:string");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs, MAIN_TENANT));
assertThat(e.getMessage()).contains("Invalid input for `string`, missing required input, but received `null`");
}
@Test
@@ -233,9 +232,9 @@ public class InputsTest {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("validatedString", "foo");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(map, "tenant4"));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant4"));
assertThat(e.getMessage()).contains( "Invalid value for input `validatedString`. Cause: it must match the pattern");
assertThat(e.getMessage()).contains("Invalid input for `validatedString`, it must match the pattern");
}
@Test
@@ -243,15 +242,15 @@ public class InputsTest {
void inputValidatedIntegerBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedInt", "9");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant5"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedInt`. Cause: it must be more than `10`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant5"));
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be more than `10`, but received `9`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedInt", "21");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant5"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant5"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedInt`. Cause: it must be less than `20`");
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be less than `20`, but received `21`");
}
@Test
@@ -259,15 +258,15 @@ public class InputsTest {
void inputValidatedDateBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDate", "2022-01-01");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant6"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDate`. Cause: it must be after `2023-01-01`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant6"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be after `2023-01-01`, but received `2022-01-01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDate", "2024-01-01");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant6"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant6"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDate`. Cause: it must be before `2023-12-31`");
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be before `2023-12-31`, but received `2024-01-01`");
}
@Test
@@ -275,15 +274,15 @@ public class InputsTest {
void inputValidatedDateTimeBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDateTime", "2022-01-01T00:00:00Z");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant7"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDateTime`. Cause: it must be after `2023-01-01T00:00:00Z`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant7"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be after `2023-01-01T00:00:00Z`, but received `2022-01-01T00:00:00Z`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDateTime", "2024-01-01T00:00:00Z");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant7"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant7"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDateTime`. Cause: it must be before `2023-12-31T23:59:59Z`");
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be before `2023-12-31T23:59:59Z`");
}
@Test
@@ -291,15 +290,15 @@ public class InputsTest {
void inputValidatedDurationBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDuration", "PT1S");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant8"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDuration`. Cause: It must be more than `PT10S`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant8"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be more than `PT10S`, but received `PT1S`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDuration", "PT30S");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant8"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant8"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedDuration`. Cause: It must be less than `PT20S`");
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be less than `PT20S`, but received `PT30S`");
}
@Test
@@ -307,15 +306,15 @@ public class InputsTest {
void inputValidatedFloatBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedFloat", "0.01");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant9"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedFloat`. Cause: it must be more than `0.1`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant9"));
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be more than `0.1`, but received `0.01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedFloat", "1.01");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant9"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant9"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedFloat`. Cause: it must be less than `0.5`");
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be less than `0.5`, but received `1.01`");
}
@Test
@@ -323,15 +322,15 @@ public class InputsTest {
void inputValidatedTimeBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedTime", "00:00:01");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMin, "tenant10"));
assertThat(e.getMessage()).contains( "Invalid value for input `validatedTime`. Cause: it must be after `01:00`");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant10"));
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be after `01:00`, but received `00:00:01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedTime", "14:00:00");
e = assertThrows(InputOutputValidationException.class, () -> typedInputs(mapMax, "tenant10"));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant10"));
assertThat(e.getMessage()).contains("Invalid value for input `validatedTime`. Cause: it must be before `11:59:59`");
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be before `11:59:59`, but received `14:00:00`");
}
@Test
@@ -340,9 +339,9 @@ public class InputsTest {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("uri", "http:/bla");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(map, "tenant11"));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant11"));
assertThat(e.getMessage()).contains( "Invalid value for input `uri`. Cause: Invalid URI format." );
assertThat(e.getMessage()).contains("Invalid input for `uri`, Expected `URI` but received `http:/bla`, but received `http:/bla`");
}
@Test
@@ -351,9 +350,9 @@ public class InputsTest {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("enum", "INVALID");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(map, "tenant12"));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant12"));
assertThat(e.getMessage()).isEqualTo("Invalid value for input `enum`. Cause: it must match the values `[ENUM_VALUE, OTHER_ONE]`");
assertThat(e.getMessage()).isEqualTo("enum: Invalid input for `enum`, it must match the values `[ENUM_VALUE, OTHER_ONE]`, but received `INVALID`");
}
@Test
@@ -362,9 +361,9 @@ public class InputsTest {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("array", "[\"s1\", \"s2\"]");
InputOutputValidationException e = assertThrows(InputOutputValidationException.class, () -> typedInputs(map, "tenant13"));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant13"));
assertThat(e.getMessage()).contains( "Invalid value for input `array`. Cause: Unable to parse array element as `INT` on `s1`");
assertThat(e.getMessage()).contains("Invalid input for `array`, Unable to parse array element as `INT` on `s1`, but received `[\"s1\", \"s2\"]`");
}
@Test
@@ -468,20 +467,7 @@ public class InputsTest {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat((String) execution.findTaskRunsByTaskId("file").getFirst().getOutputs().get("value")).isEqualTo(file.toString());
}
@Test
@LoadFlows(value = "flows/invalids/inputs-with-multiple-constraint-violations.yaml")
void multipleConstraintViolations() {
InputOutputValidationException ex = assertThrows(InputOutputValidationException.class, ()-> runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "inputs-with-multiple-constraint-violations", null,
(f, e) ->flowIO.readExecutionInputs(f, e , Map.of("multi", List.of("F", "H")) )));
List<String> messages = Arrays.asList(ex.getMessage().split(System.lineSeparator()));
assertThat(messages).containsExactlyInAnyOrder(
"Invalid value for input `multi`. Cause: you can't define both `values` and `options`",
"Invalid value for input `multi`. Cause: value `F` doesn't match the values `[A, B, C]`",
"Invalid value for input `multi`. Cause: value `H` doesn't match the values `[A, B, C]`"
);
}
private URI createFile() throws IOException {
File tempFile = File.createTempFile("file", ".txt");
Files.write(tempFile.toPath(), "Hello World".getBytes());

View File

@@ -1,6 +1,5 @@
package io.kestra.core.runners;
import io.kestra.core.exceptions.InputOutputValidationException;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.executions.Execution;
@@ -72,6 +71,6 @@ public class NoEncryptionConfiguredTest implements TestPropertyProvider {
.flowId(flow.getId())
.build();
assertThrows(InputOutputValidationException.class, () -> flowIO.readExecutionInputs(flow, execution, InputsTest.inputs));
assertThrows(ConstraintViolationException.class, () -> flowIO.readExecutionInputs(flow, execution, InputsTest.inputs));
}
}

View File

@@ -1,7 +1,6 @@
package io.kestra.plugin.core.flow;
import com.google.common.io.CharStreams;
import io.kestra.core.exceptions.InputOutputValidationException;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
@@ -329,12 +328,12 @@ public class PauseTest {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.PAUSED);
InputOutputValidationException e = assertThrows(
InputOutputValidationException.class,
ConstraintViolationException e = assertThrows(
ConstraintViolationException.class,
() -> executionService.resume(execution, flow, State.Type.RUNNING, Mono.empty(), Pause.Resumed.now()).block()
);
assertThat(e.getMessage()).contains( "Missing required input:asked");
assertThat(e.getMessage()).contains("Invalid input for `asked`, missing required input, but received `null`");
}
@SuppressWarnings("unchecked")

View File

@@ -1,18 +0,0 @@
id: inputs-with-multiple-constraint-violations
namespace: io.kestra.tests
inputs:
- id: multi
type: MULTISELECT
values:
- A
- B
- C
options:
- X
- Y
- Z
tasks:
- id: validMultiSelect
type: io.kestra.plugin.core.debug.Return
format: "{{inputs.multi}}"

View File

@@ -1,10 +0,0 @@
id: each-pause
namespace: io.kestra.tests
tasks:
- id: each_task
type: io.kestra.plugin.core.flow.ForEach
values: '["a", "b"]'
tasks:
- id: pause
type: io.kestra.plugin.core.flow.Pause

View File

@@ -1,22 +0,0 @@
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
CREATE INDEX IF NOT EXISTS logs_execution_id ON logs ("execution_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__task_id ON logs ("execution_id", "task_id");
CREATE INDEX IF NOT EXISTS logs_execution_id__taskrun_id ON logs ("execution_id", "taskrun_id");
CREATE INDEX IF NOT EXISTS logs_namespace_flow ON logs ("tenant_id", "timestamp", "level", "namespace", "flow_id");
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";
CREATE INDEX IF NOT EXISTS metrics_flow_id ON metrics ("tenant_id", "namespace", "flow_id");
CREATE INDEX IF NOT EXISTS metrics_execution_id ON metrics ("execution_id");
CREATE INDEX IF NOT EXISTS metrics_timestamp ON metrics ("tenant_id", "timestamp");

View File

@@ -1,22 +0,0 @@
ALTER TABLE logs DROP INDEX ix_execution_id;
ALTER TABLE logs DROP INDEX ix_execution_id__task_id;
ALTER TABLE logs DROP INDEX ix_execution_id__taskrun_id;
ALTER TABLE logs DROP INDEX ix_namespace_flow;
ALTER table logs drop column `deleted`;
ALTER TABLE logs ADD INDEX ix_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__task_id (`execution_id`, `task_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_execution_id__taskrun_id (`execution_id`, `taskrun_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE logs ADD INDEX ix_namespace_flow (`tenant_id`, `timestamp`, `level`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics DROP INDEX metrics_flow_id;
ALTER TABLE metrics DROP INDEX ix_metrics_execution_id;
ALTER TABLE metrics DROP INDEX metrics_timestamp;
ALTER TABLE metrics drop column `deleted`;
ALTER TABLE metrics ADD INDEX ix_metrics_flow_id (`tenant_id`, `namespace`, `flow_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_execution_id (`execution_id`), ALGORITHM=INPLACE, LOCK=NONE;
ALTER TABLE metrics ADD INDEX ix_metrics_timestamp (`tenant_id`, `timestamp`), ALGORITHM=INPLACE, LOCK=NONE;

View File

@@ -1,13 +0,0 @@
-- Indices will be re-created by the next migration
DROP INDEX logs_execution_id;
DROP INDEX logs_execution_id__task_id;
DROP INDEX logs_execution_id__taskrun_id;
DROP INDEX logs_namespace_flow;
ALTER table logs drop column "deleted";
DROP INDEX IF EXISTS metrics_flow_id;
DROP INDEX IF EXISTS metrics_execution_id;
DROP INDEX IF EXISTS metrics_timestamp;
ALTER TABLE metrics drop column "deleted";

View File

@@ -1,8 +0,0 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id ON logs (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__task_id ON logs (execution_id, task_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_execution_id__taskrun_id ON logs (execution_id, taskrun_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS logs_namespace_flow ON logs (tenant_id, timestamp, level, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_flow_id ON metrics (tenant_id, namespace, flow_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_execution_id ON metrics (execution_id);
CREATE INDEX CONCURRENTLY IF NOT EXISTS metrics_timestamp ON metrics (tenant_id, timestamp);

View File

@@ -15,11 +15,6 @@ flyway:
# We must ignore missing migrations as a V6 wrong migration was created and replaced by the V11
ignore-migration-patterns: "*:missing,*:future"
out-of-order: true
properties:
flyway:
postgresql:
transactional:
lock: false
kestra:
server-type: STANDALONE

View File

@@ -257,7 +257,10 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
.where(field("execution_id", String.class).eq(execution.getId()))
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -270,7 +273,10 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@@ -490,15 +496,5 @@ public abstract class AbstractJdbcLogRepository extends AbstractJdbcCrudReposito
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
abstract protected Field<Date> formatDateField(String dateField, DateUtils.GroupType groupType);
}

View File

@@ -185,7 +185,10 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
.where(field("execution_id", String.class).eq(execution.getId()))
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).eq(execution.getId()))
.execute();
});
}
@@ -198,21 +201,14 @@ public abstract class AbstractJdbcMetricRepository extends AbstractJdbcCrudRepos
DSLContext context = DSL.using(configuration);
return context.delete(this.jdbcRepository.getTable())
.where(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
// The deleted field is not used, so ti will always be false.
// We add it here to be sure to use the correct index.
.where(field("deleted", Boolean.class).eq(false))
.and(field("execution_id", String.class).in(executions.stream().map(Execution::getId).toList()))
.execute();
});
}
@Override
protected Condition defaultFilter(String tenantId) {
return buildTenantCondition(tenantId);
}
@Override
protected Condition defaultFilter() {
return DSL.trueCondition();
}
private List<String> queryDistinct(String tenantId, Condition condition, String field) {
return this.jdbcRepository
.getDslContextWrapper()

View File

@@ -30,7 +30,7 @@ dependencies {
// as Jackson is in the Micronaut BOM, to force its version we need to use enforcedPlatform but it didn't really work, see later :(
api enforcedPlatform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")
api enforcedPlatform("org.slf4j:slf4j-api:$slf4jVersion")
api platform("io.micronaut.platform:micronaut-platform:4.10.5")
api platform("io.micronaut.platform:micronaut-platform:4.9.4")
api platform("io.qameta.allure:allure-bom:2.32.0")
// we define cloud bom here for GCP, Azure and AWS so they are aligned for all plugins that use them (secret, storage, oss and ee plugins)
api platform('com.google.cloud:libraries-bom:26.73.0')
@@ -38,8 +38,6 @@ dependencies {
api platform('software.amazon.awssdk:bom:2.40.10')
api platform("dev.langchain4j:langchain4j-bom:$langchain4jVersion")
api platform("dev.langchain4j:langchain4j-community-bom:$langchain4jCommunityVersion")
// Micronaut 4.10 brings a Jetty version no compatible with the one from Wiremock so we bump it here
api platform("org.eclipse.jetty.ee10:jetty-ee10-bom:12.1.2")
constraints {
// downgrade to proto 1.3.2-alpha as 1.5.0 needs protobuf 4

46
ui/package-lock.json generated
View File

@@ -4220,18 +4220,18 @@
}
},
"node_modules/@shikijs/langs": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.20.0.tgz",
"integrity": "sha512-le+bssCxcSHrygCWuOrYJHvjus6zhQ2K7q/0mgjiffRbkhM4o1EWu2m+29l0yEsHDbWaWPNnDUTRVVBvBBeKaA==",
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.19.0.tgz",
"integrity": "sha512-dBMFzzg1QiXqCVQ5ONc0z2ebyoi5BKz+MtfByLm0o5/nbUu3Iz8uaTCa5uzGiscQKm7lVShfZHU1+OG3t5hgwg==",
"license": "MIT",
"dependencies": {
"@shikijs/types": "3.20.0"
"@shikijs/types": "3.19.0"
}
},
"node_modules/@shikijs/langs/node_modules/@shikijs/types": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.20.0.tgz",
"integrity": "sha512-lhYAATn10nkZcBQ0BlzSbJA3wcmL5MXUUF8d2Zzon6saZDlToKaiRX60n2+ZaHJCmXEcZRWNzn+k9vplr8Jhsw==",
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.19.0.tgz",
"integrity": "sha512-Z2hdeEQlzuntf/BZpFG8a+Fsw9UVXdML7w0o3TgSXV3yNESGon+bs9ITkQb3Ki7zxoXOOu5oJWqZ2uto06V9iQ==",
"license": "MIT",
"dependencies": {
"@shikijs/vscode-textmate": "^10.0.2",
@@ -4258,18 +4258,18 @@
}
},
"node_modules/@shikijs/themes": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.20.0.tgz",
"integrity": "sha512-U1NSU7Sl26Q7ErRvJUouArxfM2euWqq1xaSrbqMu2iqa+tSp0D1Yah8216sDYbdDHw4C8b75UpE65eWorm2erQ==",
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.19.0.tgz",
"integrity": "sha512-H36qw+oh91Y0s6OlFfdSuQ0Ld+5CgB/VE6gNPK+Hk4VRbVG/XQgkjnt4KzfnnoO6tZPtKJKHPjwebOCfjd6F8A==",
"license": "MIT",
"dependencies": {
"@shikijs/types": "3.20.0"
"@shikijs/types": "3.19.0"
}
},
"node_modules/@shikijs/themes/node_modules/@shikijs/types": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.20.0.tgz",
"integrity": "sha512-lhYAATn10nkZcBQ0BlzSbJA3wcmL5MXUUF8d2Zzon6saZDlToKaiRX60n2+ZaHJCmXEcZRWNzn+k9vplr8Jhsw==",
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.19.0.tgz",
"integrity": "sha512-Z2hdeEQlzuntf/BZpFG8a+Fsw9UVXdML7w0o3TgSXV3yNESGon+bs9ITkQb3Ki7zxoXOOu5oJWqZ2uto06V9iQ==",
"license": "MIT",
"dependencies": {
"@shikijs/vscode-textmate": "^10.0.2",
@@ -18914,6 +18914,24 @@
"hast-util-to-html": "^9.0.5"
}
},
"node_modules/shiki/node_modules/@shikijs/langs": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-3.20.0.tgz",
"integrity": "sha512-le+bssCxcSHrygCWuOrYJHvjus6zhQ2K7q/0mgjiffRbkhM4o1EWu2m+29l0yEsHDbWaWPNnDUTRVVBvBBeKaA==",
"license": "MIT",
"dependencies": {
"@shikijs/types": "3.20.0"
}
},
"node_modules/shiki/node_modules/@shikijs/themes": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-3.20.0.tgz",
"integrity": "sha512-U1NSU7Sl26Q7ErRvJUouArxfM2euWqq1xaSrbqMu2iqa+tSp0D1Yah8216sDYbdDHw4C8b75UpE65eWorm2erQ==",
"license": "MIT",
"dependencies": {
"@shikijs/types": "3.20.0"
}
},
"node_modules/shiki/node_modules/@shikijs/types": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/@shikijs/types/-/types-3.20.0.tgz",

View File

@@ -47,7 +47,7 @@
import {ref, computed, watch, onMounted, nextTick, useAttrs} from "vue";
import {useRoute} from "vue-router";
import EnterpriseBadge from "./EnterpriseBadge.vue";
import BlueprintDetail from "../override/components/flows/blueprints/BlueprintDetail.vue";
import BlueprintDetail from "./flows/blueprints/BlueprintDetail.vue";
interface Tab {
name?: string;

View File

@@ -37,31 +37,25 @@
const setupFlow = async () => {
const blueprintId = route.query.blueprintId as string;
const blueprintSource = route.query.blueprintSource as BlueprintType;
const blueprintSourceYaml = route.query.blueprintSourceYaml as string;
const implicitDefaultNamespace = authStore.user.getNamespacesForAction(
permission.FLOW,
action.CREATE,
)[0];
let flowYaml = "";
const id = getRandomID();
const selectedNamespace = (route.query.namespace as string)
?? defaultNamespace()
?? implicitDefaultNamespace
const selectedNamespace = (route.query.namespace as string)
?? defaultNamespace()
?? implicitDefaultNamespace
?? "company.team";
if (route.query.copy && flowStore.flow) {
flowYaml = flowStore.flow.source;
} else if (blueprintId && blueprintSourceYaml) {
flowYaml = blueprintSourceYaml;
} else if(blueprintId && blueprintSource === "community"){
} else if (blueprintId && blueprintSource) {
flowYaml = await blueprintsStore.getBlueprintSource({
type: blueprintSource,
kind: "flow",
id: blueprintId
});
} else if (blueprintId) {
const flowBlueprint = await blueprintsStore.getFlowBlueprint(blueprintId);
flowYaml = flowBlueprint.source;
} else {
flowYaml = `
id: ${id}

View File

@@ -90,16 +90,16 @@
import ChevronLeft from "vue-material-design-icons/ChevronLeft.vue";
import Editor from "../../../../components/inputs/Editor.vue";
import Markdown from "../../../../components/layout/Markdown.vue";
import TopNavBar from "../../../../components/layout/TopNavBar.vue";
import LowCodeEditor from "../../../../components/inputs/LowCodeEditor.vue";
import CopyToClipboard from "../../../../components/layout/CopyToClipboard.vue";
import Editor from "../../inputs/Editor.vue";
import Markdown from "../../layout/Markdown.vue";
import TopNavBar from "../../layout/TopNavBar.vue";
import LowCodeEditor from "../../inputs/LowCodeEditor.vue";
import CopyToClipboard from "../../layout/CopyToClipboard.vue";
import TaskIcon from "@kestra-io/ui-libs/src/components/misc/TaskIcon.vue";
import {useFlowStore} from "../../../../stores/flow";
import {usePluginsStore} from "../../../../stores/plugins";
import {useBlueprintsStore} from "../../../../stores/blueprints";
import {useFlowStore} from "../../../stores/flow";
import {usePluginsStore} from "../../../stores/plugins";
import {useBlueprintsStore} from "../../../stores/blueprints";
import {canCreate} from "override/composables/blueprintsPermissions";
import {parse as parseFlow} from "@kestra-io/ui-libs/flow-yaml-utils";
@@ -170,8 +170,8 @@
} else if (props.kind === "dashboard") {
additionalQuery = {
name: "home",
params: route.params?.tenant === undefined
? undefined
params: route.params?.tenant === undefined
? undefined
: JSON.stringify({tenant: route.params.tenant}),
};
}

View File

@@ -263,7 +263,7 @@
<script lang="ts">
import {ElMessage} from "element-plus";
import ValidationError from "../flows/ValidationError.vue";
import {markRaw, toRaw} from "vue";
import {toRaw} from "vue";
import {mapStores} from "pinia";
import {useExecutionsStore} from "../../stores/executions";
import debounce from "lodash/debounce";
@@ -336,10 +336,10 @@
editingArrayId: null,
editableItems: {},
// expose icon components to the template so linters and the template can resolve them
DeleteOutline: markRaw(DeleteOutline),
Pencil:markRaw(Pencil),
Plus:markRaw(Plus),
ContentSave:markRaw(ContentSave)
DeleteOutline,
Pencil,
Plus,
ContentSave
};
},
emits: ["update:modelValue", "update:modelValueNoDefault", "update:checks", "confirm", "validation"],
@@ -566,7 +566,6 @@
} else {
this.$emit("validation", {
formData: formData,
inputsMetaData: this.inputsMetaData,
callback: (response) => {
metadataCallback(response);
}

View File

@@ -35,8 +35,9 @@
import {useI18n} from "vue-i18n";
import TopNavBar from "../../../../components/layout/TopNavBar.vue";
import DottedLayout from "../../../../components/layout/DottedLayout.vue";
import BlueprintDetail from "../../../../override/components/flows/blueprints/BlueprintDetail.vue";
import BlueprintsBrowser from "../../../../override/components/flows/blueprints/BlueprintsBrowser.vue";
// @ts-expect-error - Component not typed
import BlueprintDetail from "../../../../components/flows/blueprints/BlueprintDetail.vue";
import BlueprintsBrowser from "./BlueprintsBrowser.vue";
import DemoBlueprints from "../../../../components/demo/Blueprints.vue";
import useRouteContext from "../../../../composables/useRouteContext";

View File

@@ -67,6 +67,7 @@
</div>
<div class="action-button">
<slot name="buttons" :blueprint="blueprint" />
<el-tooltip v-if="embed && !system" trigger="click" content="Copied" placement="left" :autoClose="2000" effect="light">
<el-button
type="primary"
@@ -76,11 +77,9 @@
class="p-2"
/>
</el-tooltip>
<slot name="buttons" :blueprint="{...blueprint, kind: props.blueprintKind, type: props.blueprintType}">
<el-button v-if="!embed && userCanCreate" type="primary" size="default" @click.prevent.stop="blueprintToEditor(blueprint.id)">
{{ $t('use') }}
</el-button>
</slot>
<el-button v-else-if="userCanCreate" type="primary" size="default" @click.prevent.stop="blueprintToEditor(blueprint.id)">
{{ $t('use') }}
</el-button>
</div>
</div>
</div>

View File

@@ -42,7 +42,7 @@ export default [
}
if(change) {
next({
...to,
...to,
query,
});
return;
@@ -79,7 +79,7 @@ export default [
//Blueprints
{name: "blueprints", path: "/:tenant?/blueprints/:kind/:tab", component: () => import("override/components/flows/blueprints/Blueprints.vue"), props: true},
{name: "blueprints/view", path: "/:tenant?/blueprints/:kind/:tab/:blueprintId", component: () => import("../override/components/flows/blueprints/BlueprintDetail.vue"), props: true},
{name: "blueprints/view", path: "/:tenant?/blueprints/:kind/:tab/:blueprintId", component: () => import("../components/flows/blueprints/BlueprintDetail.vue"), props: true},
//Documentation
{name: "plugins/list", path: "/:tenant?/plugins", component: () => import("../components/plugins/Plugin.vue")},

View File

@@ -24,31 +24,6 @@ interface Blueprint {
[key: string]: any;
}
export interface TemplateArgument {
id: string,
displayName: string,
type: string,
itemType?: string,
required: boolean,
defaults?: any
}
export interface BlueprintTemplate {
source: string;
templateArguments: Record<string, TemplateArgument>;
}
export interface FlowBlueprint {
id: string,
title: string,
description: string,
includedTasks?: string[],
tags?: string[],
source: string,
publishedAt?: string,
template?: BlueprintTemplate
}
const API_URL = "https://api.kestra.io/v1";
const VALIDATE = {validateStatus: (status: number) => status === 200 || status === 401};
@@ -120,54 +95,6 @@ export const useBlueprintsStore = defineStore("blueprints", () => {
return response.data;
};
const getFlowBlueprint = async (id: string): Promise<FlowBlueprint> => {
const url = `${apiUrl()}/blueprints/flow/${id}`;
const response = await axios.get(url);
if (response.data?.id) {
trackBlueprintSelection(response.data.id);
}
blueprint.value = response.data;
return response.data;
};
const createFlowBlueprint = async (toCreate: {source: string, title: string, description: string, tags: string[]}): Promise<FlowBlueprint> => {
const url = `${apiUrl()}/blueprints/flows`;
const body = {
...toCreate
}
const response = await axios.post(url, body);
return response.data;
};
const updateFlowBlueprint = async (id: string, toUpdate: {source: string, title: string, description: string, tags: string[]}) :Promise<FlowBlueprint> => {
const url = `${apiUrl()}/blueprints/flows/${id}`;
const body = {
...toUpdate
}
const response = await axios.put(url, body);
return response.data;
};
const deleteFlowBlueprint = async (idToDelete: string) => {
const url = `${apiUrl()}/blueprints/flows/${idToDelete}`;
await axios.delete(url);
};
const useFlowBlueprintTemplate = async (id: string, inputs: Record<string, object>): Promise<{generatedFlowSource: string}> => {
const url = `${apiUrl()}/blueprints/flows/${id}/use-template`;
const body = {
templateArgumentsInputs: inputs
}
const response = await axios.post(url, body);
return response.data;
}
return {
blueprint,
blueprints,
@@ -179,10 +106,5 @@ export const useBlueprintsStore = defineStore("blueprints", () => {
getBlueprintSource,
getBlueprintGraph,
getBlueprintTags,
useFlowBlueprintTemplate,
getFlowBlueprint,
createFlowBlueprint,
updateFlowBlueprint,
deleteFlowBlueprint,
};
});

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "Beenden",
"kill only parents": "Nur aktuellen beenden",
"kill parents and subflow": "Aktuelle und Subflows beenden",
"kill only parents": "Nur Eltern beenden",
"kill parents and subflow": "Eltern und Subflows beenden",
"killed confirm": "Sind Sie sicher, dass Sie die Ausführung <code>{id}</code> beenden möchten?",
"killed done": "Ausführung ist zum Beenden eingereiht",
"kv": {

View File

@@ -231,8 +231,8 @@
"mark as": "Mark as <code>{status}</code>",
"unqueue as": "Unqueue as <code>{status}</code>",
"kill": "Kill",
"kill parents and subflow": "Kill current and subflows",
"kill only parents": "Kill current only",
"kill parents and subflow": "Kill parents and subflows",
"kill only parents": "Kill parents only",
"killed confirm": "Are you sure you want to kill execution <code>{id}</code>?",
"killed done": "Execution is queued for killing",
"resume": "Resume",

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "Terminar",
"kill only parents": "Matar solo el actual",
"kill parents and subflow": "Terminar el flujo actual y los subflows",
"kill only parents": "Terminar solo padres",
"kill parents and subflow": "Terminar padres y subflows",
"killed confirm": "¿Estás seguro de terminar la ejecución <code>{id}</code>?",
"killed done": "La ejecución está en cola para ser terminada",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Clé",
"kill": "Arrêter",
"kill only parents": "Tuer uniquement le courant",
"kill parents and subflow": "Tuer le flow actuel et les subflows",
"kill only parents": "Arrêter uniquement les exécutions parents",
"kill parents and subflow": "Arrêter les exécutions parents et enfants",
"killed confirm": "Êtes-vous sur de vouloir arrêter l'exécution <code>{id}</code> ?",
"killed done": "L'exécution est en attente d'arrêt",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "केस्ट्रा",
"key": "Key",
"kill": "समाप्त करें",
"kill only parents": "केवल वर्तमान को समाप्त करें",
"kill parents and subflow": "वर्तमान और subflows को समाप्त करें",
"kill only parents": "केवल parents समाप्त करें",
"kill parents and subflow": "parents और subflows समाप्त करें",
"killed confirm": "क्या आप वाकई निष्पादन <code>{id}</code> को समाप्त करना चाहते हैं?",
"killed done": "निष्पादन समाप्त करने के लिए कतारबद्ध है",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "Termina",
"kill only parents": "Uccidi solo corrente",
"kill parents and subflow": "Uccidi il flow corrente e i subflow",
"kill only parents": "Termina solo genitori",
"kill parents and subflow": "Termina genitori e subflows",
"killed confirm": "Sei sicuro di voler terminare l'esecuzione <code>{id}</code>?",
"killed done": "L'esecuzione è in coda per la terminazione",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "キル",
"kill only parents": "現在のもののみを終了",
"kill parents and subflow": "現在のflowとsubflowを終了",
"kill only parents": "のみをキル",
"kill parents and subflow": "とsubflowsをキル",
"killed confirm": "実行<code>{id}</code>をキルしてもよろしいですか?",
"killed done": "実行はキルのためにキューに入れられました",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "종료",
"kill only parents": "현재 것만 종료",
"kill parents and subflow": "현재 및 subflow 종료",
"kill only parents": "부모만 종료",
"kill parents and subflow": "부모 및 subflows 종료",
"killed confirm": "실행 <code>{id}</code>을(를) 종료하시겠습니까?",
"killed done": "종료 중입니다.",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Klucz",
"kill": "Zakończ",
"kill only parents": "Zabij tylko bieżący",
"kill parents and subflow": "Zatrzymaj bieżący flow i subflow",
"kill only parents": "Zakończ tylko rodziców",
"kill parents and subflow": "Zakończ rodziców i subflows",
"killed confirm": "Czy na pewno chcesz zakończyć wykonanie <code>{id}</code>?",
"killed done": "Wykonanie jest w kolejce do zakończenia",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "Matar",
"kill only parents": "Matar apenas o atual",
"kill parents and subflow": "Matar o flow atual e subflows",
"kill only parents": "Matar apenas pais",
"kill parents and subflow": "Matar pais e subflows",
"killed confirm": "Tem certeza de que deseja matar a execução <code>{id}</code>?",
"killed done": "Execução está na fila para ser morta",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Key",
"kill": "Matar",
"kill only parents": "Matar apenas o atual",
"kill parents and subflow": "Matar o flow atual e subflows",
"kill only parents": "Matar apenas pais",
"kill parents and subflow": "Matar pais e subflows",
"killed confirm": "Tem certeza de que deseja matar a execução <code>{id}</code>?",
"killed done": "Execução está na fila para ser morta",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "Ключ",
"kill": "Убить",
"kill only parents": "Убить только текущий",
"kill parents and subflow": "Прервать текущий и subflows",
"kill only parents": "Убить только родителей",
"kill parents and subflow": "Убить родителей и подпроцессы",
"killed confirm": "Вы уверены, что хотите убить выполнение <code>{id}</code>?",
"killed done": "Выполнение поставлено в очередь на убийство",
"kv": {

View File

@@ -1027,8 +1027,8 @@
"kestra": "Kestra",
"key": "键",
"kill": "终止",
"kill only parents": "仅终止当前",
"kill parents and subflow": "终止当前和subflows",
"kill only parents": "仅终止父流程",
"kill parents and subflow": "终止父流程和子流程",
"killed confirm": "确定要终止执行 <code>{id}</code> 吗?",
"killed done": "执行已排队终止",
"kv": {

View File

@@ -19,10 +19,6 @@ dependencies {
implementation "io.micronaut:micronaut-management"
implementation "io.micronaut:micronaut-http-client"
implementation "io.micronaut:micronaut-http-server-netty"
// See https://github.com/netty-contrib/codec-multipart/pull/25
// See https://github.com/kestra-io/kestra/issues/9743
// There is an issue on Netty multipart content decoding that is fixed in 5.x, this library will bring the same fix for 4.x
implementation("io.netty.contrib:netty-codec-multipart-vintage:1.0.0.Alpha1")
implementation "io.micronaut.cache:micronaut-cache-core"
implementation "io.micronaut.cache:micronaut-cache-caffeine"

View File

@@ -36,10 +36,6 @@ public class ErrorController {
public HttpResponse<JsonError> error(HttpRequest<?> request, JsonParseException e) {
return jsonError(request, e, HttpStatus.UNPROCESSABLE_ENTITY, "Invalid json");
}
@Error(global = true)
public HttpResponse<JsonError> error(HttpRequest<?> request, InputOutputValidationException e) {
return jsonError(request, e, HttpStatus.UNPROCESSABLE_ENTITY, "Invalid entity");
}
@Error(global = true)
public HttpResponse<JsonError> error(HttpRequest<?> request, ConversionErrorException e) {

View File

@@ -2670,12 +2670,7 @@ public class ExecutionController {
) {
}
public static ApiValidateExecutionInputsResponse of(
String id,
String namespace,
List<Check> checks,
List<InputAndValue> inputs
) {
public static ApiValidateExecutionInputsResponse of(String id, String namespace, List<Check> checks, List<InputAndValue> inputs) {
return new ApiValidateExecutionInputsResponse(
id,
namespace,
@@ -2684,21 +2679,14 @@ public class ExecutionController {
it.value(),
it.enabled(),
it.isDefault(),
// Map the Set<InputOutputValidationException> to ApiInputError
Optional.ofNullable(it.exceptions())
.map(exSet -> exSet.stream()
.map(e -> new ApiInputError(e.getMessage()))
Optional.ofNullable(it.exception()).map(exception ->
exception.getConstraintViolations()
.stream()
.map(cv -> new ApiInputError(cv.getMessage()))
.toList()
)
.orElse(List.of())
).orElse(List.of())
)).toList(),
checks.stream()
.map(check -> new ApiCheckFailure(
check.getMessage(),
check.getStyle(),
check.getBehavior()
))
.toList()
checks.stream().map(check -> new ApiCheckFailure(check.getMessage(), check.getStyle(), check.getBehavior())).toList()
);
}
}

View File

@@ -57,7 +57,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -140,7 +140,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -195,7 +195,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -246,7 +246,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -330,7 +330,7 @@ class DashboardControllerTest {
timeWindow:
default: P30D # P30DT30H
max: P365D
charts:
- id: logs_timeseries
type: io.kestra.plugin.core.dashboard.chart.TimeSeries
@@ -378,6 +378,7 @@ class DashboardControllerTest {
.namespace(fakeNamespace)
.level(Level.INFO)
.attemptNumber(1)
.deleted(false)
.executionId(fakeExecutionId)
.tenantId(MAIN_TENANT)
.executionKind(ExecutionKind.NORMAL)
@@ -446,6 +447,7 @@ class DashboardControllerTest {
.namespace(fakeNamespace)
.level(Level.INFO)
.attemptNumber(1)
.deleted(false)
.executionId(fakeExecutionId)
.tenantId(MAIN_TENANT)
.executionKind(ExecutionKind.NORMAL)

View File

@@ -242,7 +242,7 @@ class ExecutionControllerRunnerTest {
String response = e.getResponse().getBody(String.class).orElseThrow();
assertThat(response).contains("Invalid entity");
assertThat(response).contains("Invalid value for input `validatedString`");
assertThat(response).contains("Invalid input for `validatedString`");
}
@Test
@@ -1071,7 +1071,7 @@ class ExecutionControllerRunnerTest {
.contentType(MediaType.MULTIPART_FORM_DATA_TYPE)
));
assertThat(exception.getStatus().getCode()).isEqualTo(422);
assertThat(exception.getMessage()).isEqualTo("Invalid entity: Missing required input:asked");
assertThat(exception.getMessage()).isEqualTo("Invalid entity: asked: Invalid input for `asked`, missing required input, but received `null`");
}
@Test

View File

@@ -57,6 +57,7 @@ public class NamespaceControllerTest {
);
assertThat(namespace.getId()).isEqualTo("my.ns");
assertThat(namespace.isDeleted()).isFalse();
}
@SuppressWarnings("unchecked")