jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: migrate AbstractExecScript, CommandsWrapper, NamespaceFiles, OutputFilesInterface to dynamic properties

Browse Source
This commit is contained in:
Mathieu Gabelle
2024-12-19 16:50:10 +01:00
committed by Mathieu Gabelle
parent 316d514fa3
commit c6a09cd9de
5 changed files with 38 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/src/main/java/io/kestra/core/models/tasks/NamespaceFiles.java
View File

@@ -1,6 +1,7 @@
package io.kestra.core.models.tasks;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.micronaut.core.annotation.Introspected;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
@@ -20,21 +21,18 @@ public class NamespaceFiles {
@Schema(
title = "Whether to enable namespace files to be loaded into the working directory. If explicitly set to `true` in a task, it will load all [Namespace Files](https://kestra.io/docs/developer-guide/namespace-files) into the task's working directory. Note that this property is by default set to `true` so that you can specify only the `include` and `exclude` properties to filter the files to load without having to explicitly set `enabled` to `true`."
)
@PluginProperty
@Builder.Default
private Boolean enabled = true;
private Property<Boolean> enabled = Property.of(true);
@Schema(
title = "A list of filters to include only matching glob patterns. This allows you to only load a subset of the [Namespace Files](https://kestra.io/docs/developer-guide/namespace-files) into the working directory."
)
@PluginProperty
@Valid
private List<String> include;
private Property<List<String>> include;
@Schema(
title = "A list of filters to exclude matching glob patterns. This allows you to exclude a subset of the [Namespace Files](https://kestra.io/docs/developer-guide/namespace-files) from being downloaded at runtime. You can combine this property together with `include` to only inject a subset of files that you need into the task's working directory."
)
@PluginProperty
@Valid
private List<String> exclude;
private Property<List<String>> exclude;
}

4
core/src/main/java/io/kestra/core/models/tasks/OutputFilesInterface.java
View File

@@ -1,6 +1,7 @@
package io.kestra.core.models.tasks;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import java.util.List;
@@ -10,6 +11,5 @@ public interface OutputFilesInterface {
title = "The files from the local filesystem to send to Kestra's internal storage.",
description = "Must be a list of [glob](https://en.wikipedia.org/wiki/Glob_(programming)) expressions relative to the current working directory, some examples: `my-dir/**`, `my-dir/*/**` or `my-dir/my-file.txt`."
)
@PluginProperty(dynamic = true)
List<String> getOutputFiles();
Property<List<String>> getOutputFiles();
}

12
core/src/main/java/io/kestra/plugin/core/flow/WorkingDirectory.java
View File

@@ -9,6 +9,7 @@ import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.NextTaskRun;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.InputFilesInterface;
import io.kestra.core.models.tasks.NamespaceFiles;
import io.kestra.core.models.tasks.NamespaceFilesInterface;
@@ -202,7 +203,7 @@ public class WorkingDirectory extends Sequential implements NamespaceFilesInterf
private Object inputFiles;
private List<String> outputFiles;
private Property<List<String>> outputFiles;
@Override
public List<NextTaskRun> resolveNexts(RunContext runContext, Execution execution, TaskRun parentTaskRun) throws IllegalVariableEvaluationException {
@@ -262,10 +263,13 @@ public class WorkingDirectory extends Sequential implements NamespaceFilesInterf
}
}
if (this.namespaceFiles != null && !Boolean.FALSE.equals(this.namespaceFiles.getEnabled())) {
if (this.namespaceFiles != null && !Boolean.FALSE.equals(runContext.render(this.namespaceFiles.getEnabled()).as(Boolean.class).orElse(true))) {
runContext.storage()
.namespace()
.findAllFilesMatching(this.namespaceFiles.getInclude(), this.namespaceFiles.getExclude())
.findAllFilesMatching(
runContext.render(this.namespaceFiles.getInclude()).asList(String.class),
runContext.render(this.namespaceFiles.getExclude()).asList(String.class)
)
.forEach(Rethrow.throwConsumer(namespaceFile -> {
InputStream content = runContext.storage().getFile(namespaceFile.uri());
runContext.workingDir().putFile(Path.of(namespaceFile.path()), content);
@@ -280,7 +284,7 @@ public class WorkingDirectory extends Sequential implements NamespaceFilesInterf
public void postExecuteTasks(RunContext runContext, TaskRun taskRun) throws Exception {
if (this.outputFiles != null) {
try {
Map<String, URI> outputFilesURIs = FilesService.outputFiles(runContext, this.outputFiles);
Map<String, URI> outputFilesURIs = FilesService.outputFiles(runContext, runContext.render(this.outputFiles).asList(String.class));
if (!outputFilesURIs.isEmpty()) {
final ByteArrayOutputStream os = new ByteArrayOutputStream();
try (os) {

43
script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java
View File

@@ -2,6 +2,7 @@ package io.kestra.plugin.scripts.exec;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.*;
import io.kestra.core.models.tasks.runners.TargetOS;
import io.kestra.core.models.tasks.runners.TaskRunner;
@@ -22,6 +23,7 @@ import org.apache.commons.lang3.SystemUtils;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,26 +56,20 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
@Schema(
title = "A list of commands that will run before the `commands`, allowing to set up the environment e.g. `pip install -r requirements.txt`."
)
@PluginProperty(dynamic = true)
protected List<String> beforeCommands;
protected Property<List<String>> beforeCommands;
@Schema(
title = "Additional environment variables for the current process."
)
@PluginProperty(
additionalProperties = String.class,
dynamic = true
)
protected Map<String, String> env;
protected Property<Map<String, String>> env;
@Builder.Default
@Schema(
title = "Whether to set the task state to `WARNING` when any `stdErr` output is detected.",
description = "Note that a script error will set the state to `FAILED` regardless."
)
@PluginProperty
@NotNull
protected Boolean warningOnStdErr = true;
protected Property<Boolean> warningOnStdErr = Property.of(true);
@Builder.Default
@Schema(
@@ -90,14 +86,13 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
description = "If set to `false` all commands will be executed one after the other. The final state of task execution is determined by the last command. Note that this property maybe be ignored if a non compatible interpreter is specified." +
"\nYou can also disable it if your interpreter does not support the `set -e`option."
)
@PluginProperty
protected Boolean failFast = true;
protected Property<Boolean> failFast = Property.of(true);
private NamespaceFiles namespaceFiles;
private Object inputFiles;
private List<String> outputFiles;
private Property<List<String>> outputFiles;
@Schema(
title = "Whether to setup the output directory mechanism.",
@@ -112,7 +107,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
title = "The target operating system where the script will run."
)
@Builder.Default
protected TargetOS targetOS = TargetOS.AUTO;
protected Property<TargetOS> targetOS = Property.of(TargetOS.AUTO);
@Schema(
title = "Deprecated - use the 'taskRunner' property instead.",
@@ -125,8 +120,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
@Schema(
title = "The task runner container image, only used if the task runner is container-based."
)
@PluginProperty(dynamic = true)
public abstract String getContainerImage();
public abstract Property<String> getContainerImage();
/**
* Allow setting Docker options defaults values.
@@ -151,27 +145,28 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
runContext.logger().debug("Using task runner '{}'", this.getTaskRunner().getType());
}
Map<String, String> renderedEnv = runContext.render(this.getEnv()).asMap(String.class, String.class);
return new CommandsWrapper(runContext)
.withEnv(this.getEnv())
.withWarningOnStdErr(this.getWarningOnStdErr())
.withEnv(renderedEnv.isEmpty() ? new HashMap<>() : renderedEnv)
.withWarningOnStdErr(runContext.render(this.getWarningOnStdErr()).as(Boolean.class).orElseThrow())
.withRunnerType(this.getRunner())
.withContainerImage(runContext.render(this.getContainerImage()))
.withContainerImage(runContext.render(this.getContainerImage()).as(String.class).orElse(null))
.withTaskRunner(this.getTaskRunner())
.withDockerOptions(this.getDocker() != null ? this.injectDefaults(this.getDocker()) : null)
.withNamespaceFiles(this.getNamespaceFiles())
.withInputFiles(this.getInputFiles())
.withOutputFiles(this.getOutputFiles())
.withOutputFiles(runContext.render(this.getOutputFiles()).asList(String.class))
.withEnableOutputDirectory(this.getOutputDirectory())
.withTimeout(runContext.render(this.getTimeout()).as(Duration.class).orElse(null))
.withTargetOS(this.getTargetOS());
.withTargetOS(runContext.render(this.getTargetOS()).as(TargetOS.class).orElseThrow());
}
protected List<String> getBeforeCommandsWithOptions() {
return mayAddExitOnErrorCommands(this.getBeforeCommands());
protected List<String> getBeforeCommandsWithOptions(RunContext runContext) throws IllegalVariableEvaluationException {
return mayAddExitOnErrorCommands(runContext.render(this.getBeforeCommands()).asList(String.class), runContext);
}
protected List<String> mayAddExitOnErrorCommands(List<String> commands) {
if (!this.getFailFast()) {
protected List<String> mayAddExitOnErrorCommands(List<String> commands, RunContext runContext) throws IllegalVariableEvaluationException {
if (!runContext.render(this.getFailFast()).as(Boolean.class).orElseThrow()) {
return commands;
}

7
script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java
View File

@@ -132,11 +132,14 @@ public class CommandsWrapper implements TaskCommands {
}
public ScriptOutput run() throws Exception {
if (this.namespaceFiles != null && !Boolean.FALSE.equals(this.namespaceFiles.getEnabled())) {
if (this.namespaceFiles != null && !Boolean.FALSE.equals(runContext.render(this.namespaceFiles.getEnabled()).as(Boolean.class).orElse(true))) {
List<NamespaceFile> matchedNamespaceFiles = runContext.storage()
.namespace()
.findAllFilesMatching(this.namespaceFiles.getInclude(), this.namespaceFiles.getExclude());
.findAllFilesMatching(
runContext.render(this.namespaceFiles.getInclude()).asList(String.class),
runContext.render(this.namespaceFiles.getExclude()).asList(String.class)
);
matchedNamespaceFiles.forEach(Rethrow.throwConsumer(namespaceFile -> {
InputStream content = runContext.storage().getFile(namespaceFile.uri());