jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: migrate plugin.core.log to dynamic properties (#6823)

Browse Source
This commit is contained in:
Mathieu Gabelle
2025-01-20 13:30:04 +01:00
committed by GitHub
parent 4b2c10d6e2
commit 25ca9f74e5
8 changed files with 66 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java
View File

@@ -75,10 +75,9 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
@Schema(
title = "Which interpreter to use."
)
@PluginProperty
@PluginProperty(dynamic = true)
@NotNull
@NotEmpty
protected List<String> interpreter = List.of("/bin/sh", "-c");
protected Property<List<String>> interpreter = Property.of(List.of("/bin/sh", "-c"));
@Builder.Default
@Schema(
@@ -101,7 +100,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
deprecated = true
)
@Deprecated
private Boolean outputDirectory;
private Property<Boolean> outputDirectory;
@Schema(
title = "The target operating system where the script will run."
@@ -156,7 +155,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
.withNamespaceFiles(this.getNamespaceFiles())
.withInputFiles(this.getInputFiles())
.withOutputFiles(runContext.render(this.getOutputFiles()).asList(String.class))
.withEnableOutputDirectory(this.getOutputDirectory())
.withEnableOutputDirectory(runContext.render(this.getOutputDirectory()).as(Boolean.class).orElse(null))
.withTimeout(runContext.render(this.getTimeout()).as(Duration.class).orElse(null))
.withTargetOS(runContext.render(this.getTargetOS()).as(TargetOS.class).orElseThrow());
}

7
script/src/main/java/io/kestra/plugin/scripts/exec/scripts/models/DockerOptions.java
View File

@@ -1,6 +1,7 @@
package io.kestra.plugin.scripts.exec.scripts.models;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.plugin.scripts.runner.docker.*;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Builder;
@@ -77,9 +78,8 @@ public class DockerOptions {
@PluginProperty(dynamic = true)
protected List<String> volumes;
@PluginProperty
@Builder.Default
protected PullPolicy pullPolicy = PullPolicy.ALWAYS;
protected Property<PullPolicy> pullPolicy = Property.of(PullPolicy.ALWAYS);
@Schema(
title = "A list of device requests to be sent to device drivers."
@@ -116,8 +116,7 @@ public class DockerOptions {
@Schema(
title = "Give extended privileges to this container."
)
@PluginProperty(dynamic = true)
private Boolean privileged;
private Property<Boolean> privileged;
@Deprecated
public void setDockerHost(String host) {

5
script/src/main/java/io/kestra/plugin/scripts/runner/docker/Cpu.java
View File

@@ -1,6 +1,6 @@
package io.kestra.plugin.scripts.runner.docker;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -14,6 +14,5 @@ public class Cpu {
title = "The maximum amount of CPU resources a container can use.",
description = "Make sure to set that to a numeric value e.g. `cpus: \"1.5\"` or `cpus: \"4\"` or For instance, if the host machine has two CPUs and you set `cpus: \"1.5\"`, the container is guaranteed **at most** one and a half of the CPUs."
)
@PluginProperty
private Long cpus;
private Property<Long> cpus;
}

20
script/src/main/java/io/kestra/plugin/scripts/runner/docker/Credentials.java
View File

@@ -1,6 +1,6 @@
package io.kestra.plugin.scripts.runner.docker;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -17,37 +17,31 @@ public class Credentials {
title = "The registry URL.",
description = "If not defined, the registry will be extracted from the image name."
)
@PluginProperty(dynamic = true)
private String registry;
private Property<String> registry;
@Schema(
title = "The registry username."
)
@PluginProperty(dynamic = true)
private String username;
private Property<String> username;
@Schema(
title = "The registry password."
)
@PluginProperty(dynamic = true)
private String password;
private Property<String> password;
@Schema(
title = "The registry token."
)
@PluginProperty(dynamic = true)
private String registryToken;
private Property<String> registryToken;
@Schema(
title = "The identity token."
)
@PluginProperty(dynamic = true)
private String identityToken;
private Property<String> identityToken;
@Schema(
title = "The registry authentication.",
description = "The `auth` field is a base64-encoded authentication string of `username:password` or a token."
)
@PluginProperty(dynamic = true)
private String auth;
private Property<String> auth;
}

17
script/src/main/java/io/kestra/plugin/scripts/runner/docker/DeviceRequest.java
View File

@@ -1,6 +1,6 @@
package io.kestra.plugin.scripts.runner.docker;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -16,25 +16,20 @@ import java.util.Map;
title = "A request for devices to be sent to device drivers."
)
public class DeviceRequest {
@PluginProperty(dynamic = true)
private String driver;
private Property<String> driver;
@PluginProperty
private Integer count;
private Property<Integer> count;
@PluginProperty(dynamic = true)
private List<String> deviceIds;
private Property<List<String>> deviceIds;
@Schema(
title = "A list of capabilities; an OR list of AND lists of capabilities."
)
@PluginProperty
private List<List<String>> capabilities;
private Property<List<List<String>>> capabilities;
@Schema(
title = "Driver-specific options, specified as key/value pairs.",
description = "These options are passed directly to the driver."
)
@PluginProperty
private Map<String, String> options;
private Property<Map<String, String>> options;
}

66
script/src/main/java/io/kestra/plugin/scripts/runner/docker/Docker.java
View File

@@ -14,6 +14,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.retrys.Exponential;
import io.kestra.core.models.tasks.runners.*;
import io.kestra.core.runners.DefaultRunContext;
@@ -237,9 +238,8 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
Use the `ALWAYS` pull policy to pull the latest version of an image
even if an image with the same tag already exists."""
)
@PluginProperty
@Builder.Default
protected PullPolicy pullPolicy = PullPolicy.ALWAYS;
protected Property<PullPolicy> pullPolicy = Property.of(PullPolicy.ALWAYS);
@Schema(
title = "A list of device requests to be sent to device drivers."
@@ -276,8 +276,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
@Schema(
title = "Give extended privileges to this container."
)
@PluginProperty(dynamic = true)
private Boolean privileged;
private Property<Boolean> privileged;
@Schema(
title = "File handling strategy.",
@@ -288,23 +287,20 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
)
@NotNull
@Builder.Default
@PluginProperty
private FileHandlingStrategy fileHandlingStrategy = FileHandlingStrategy.VOLUME;
private Property<FileHandlingStrategy> fileHandlingStrategy = Property.of(FileHandlingStrategy.VOLUME);
@Schema(
title = "Whether the container should be deleted upon completion."
)
@NotNull
@Builder.Default
@PluginProperty
private Boolean delete = true;
private Property<Boolean> delete = Property.of(true);
@Builder.Default
@Schema(
title = "Whether to wait for the container to exit."
)
@PluginProperty
private final Boolean wait = true;
private final Property<Boolean> wait = Property.of(true);
/**
* Convenient default instance to be used as task default value for a 'taskRunner' property.
@@ -357,8 +353,9 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
String resolvedHost = DockerService.findHost(runContext, this.host);
try (DockerClient dockerClient = dockerClient(runContext, image, resolvedHost)) {
// pull image
if (this.getPullPolicy() != PullPolicy.NEVER) {
pullImage(dockerClient, image, this.getPullPolicy(), logger);
var renderedPolicy = runContext.render(this.getPullPolicy()).as(PullPolicy.class).orElseThrow();
if (!PullPolicy.NEVER.equals(renderedPolicy)) {
pullImage(dockerClient, image, renderedPolicy, logger);
}
// create container
@@ -376,7 +373,8 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
String filesVolumeName = null;
// create a volume if we need to handle files
if (needVolume && this.fileHandlingStrategy == FileHandlingStrategy.VOLUME) {
var strategy = runContext.render(this.fileHandlingStrategy).as(FileHandlingStrategy.class).orElse(null);
if (needVolume && FileHandlingStrategy.VOLUME.equals(strategy)) {
CreateVolumeCmd files = dockerClient.createVolumeCmd()
.withLabels(ScriptService.labels(runContext, "kestra.io/"));
filesVolumeName = files.exec().getName();
@@ -437,7 +435,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
);
}
if (!wait) {
if (!Boolean.TRUE.equals(runContext.render(wait).as(Boolean.class).orElseThrow())) {
return TaskRunnerResult.<DockerTaskRunnerDetailResult>builder()
.exitCode(0)
.logConsumer(defaultLogConsumer)
@@ -513,7 +511,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
Await.until(ended::get);
if (exitCode != 0) {
if (needVolume && this.fileHandlingStrategy == FileHandlingStrategy.VOLUME && filesVolumeName != null) {
if (needVolume && FileHandlingStrategy.VOLUME.equals(strategy) && filesVolumeName != null) {
downloadOutputFiles(exec.getId(), dockerClient, runContext, taskCommands);
}
@@ -522,7 +520,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
logger.debug("Command succeed with exit code {}", exitCode);
}
if (needVolume && this.fileHandlingStrategy == FileHandlingStrategy.VOLUME && filesVolumeName != null) {
if (needVolume && FileHandlingStrategy.VOLUME.equals(strategy) && filesVolumeName != null) {
downloadOutputFiles(exec.getId(), dockerClient, runContext, taskCommands);
}
@@ -537,13 +535,13 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
// come to a normal end.
kill();
if (Boolean.TRUE.equals(delete)) {
if (Boolean.TRUE.equals(runContext.render(delete).as(Boolean.class).orElseThrow())) {
dockerClient.removeContainerCmd(exec.getId()).exec();
if (logger.isTraceEnabled()) {
logger.trace("Container deleted: {}", exec.getId());
}
if (needVolume && this.fileHandlingStrategy == FileHandlingStrategy.VOLUME && filesVolumeName != null) {
if (needVolume && FileHandlingStrategy.VOLUME.equals(strategy) && filesVolumeName != null) {
dockerClient.removeVolumeCmd(filesVolumeName).exec();
if (logger.isTraceEnabled()) {
@@ -694,7 +692,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
}
List<Bind> binds = new ArrayList<>();
if (this.fileHandlingStrategy == FileHandlingStrategy.MOUNT && workingDirectory != null) {
if (FileHandlingStrategy.MOUNT.equals(runContext.render(this.fileHandlingStrategy).as(FileHandlingStrategy.class).orElse(null)) && workingDirectory != null) {
String bindPath = windowsToUnixPath(workingDirectory.toString());
binds.add(new Bind(
bindPath,
@@ -717,45 +715,43 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
.getDeviceRequests()
.stream()
.map(throwFunction(deviceRequest -> new com.github.dockerjava.api.model.DeviceRequest()
.withDriver(runContext.render(deviceRequest.getDriver()))
.withCount(deviceRequest.getCount())
.withDeviceIds(runContext.render(deviceRequest.getDeviceIds()))
.withCapabilities(deviceRequest.getCapabilities())
.withOptions(deviceRequest.getOptions())
.withDriver(runContext.render(deviceRequest.getDriver()).as(String.class).orElse(null))
.withCount(runContext.render(deviceRequest.getCount()).as(Integer.class).orElse(null))
.withDeviceIds(runContext.render(deviceRequest.getDeviceIds()).asList(String.class))
.withCapabilities(runContext.render(deviceRequest.getCapabilities()).asList(List.class))
.withOptions(runContext.render(deviceRequest.getOptions()).asMap(String.class, String.class))
))
.toList()
);
}
if (this.getCpu() != null) {
if (this.getCpu().getCpus() != null) {
hostConfig.withCpuQuota(this.getCpu().getCpus() * 10000L);
}
if (this.getCpu() != null && this.getCpu().getCpus() != null) {
hostConfig.withCpuQuota(runContext.render(this.getCpu().getCpus()).as(Long.class).orElseThrow() * 10000L);
}
if (this.getMemory() != null) {
if (this.getMemory().getMemory() != null) {
hostConfig.withMemory(convertBytes(runContext.render(this.getMemory().getMemory())));
hostConfig.withMemory(convertBytes(runContext.render(this.getMemory().getMemory()).as(String.class).orElse(null)));
}
if (this.getMemory().getMemorySwap() != null) {
hostConfig.withMemorySwap(convertBytes(runContext.render(this.getMemory().getMemorySwap())));
hostConfig.withMemorySwap(convertBytes(runContext.render(this.getMemory().getMemorySwap()).as(String.class).orElse(null)));
}
if (this.getMemory().getMemorySwappiness() != null) {
hostConfig.withMemorySwappiness(convertBytes(runContext.render(this.getMemory().getMemorySwappiness())));
hostConfig.withMemorySwappiness(convertBytes(runContext.render(this.getMemory().getMemorySwappiness()).as(String.class).orElse(null)));
}
if (this.getMemory().getMemoryReservation() != null) {
hostConfig.withMemoryReservation(convertBytes(runContext.render(this.getMemory().getMemoryReservation())));
hostConfig.withMemoryReservation(convertBytes(runContext.render(this.getMemory().getMemoryReservation()).as(String.class).orElse(null)));
}
if (this.getMemory().getKernelMemory() != null) {
hostConfig.withKernelMemory(convertBytes(runContext.render(this.getMemory().getKernelMemory())));
hostConfig.withKernelMemory(convertBytes(runContext.render(this.getMemory().getKernelMemory()).as(String.class).orElse(null)));
}
if (this.getMemory().getOomKillDisable() != null) {
hostConfig.withOomKillDisable(this.getMemory().getOomKillDisable());
hostConfig.withOomKillDisable(runContext.render(this.getMemory().getOomKillDisable()).as(Boolean.class).orElse(null));
}
}
@@ -764,7 +760,7 @@ public class Docker extends TaskRunner<Docker.DockerTaskRunnerDetailResult> {
}
if (this.getPrivileged() != null) {
hostConfig.withPrivileged(this.getPrivileged());
hostConfig.withPrivileged(runContext.render(this.getPrivileged()).as(Boolean.class).orElseThrow());
}
if (this.getNetworkMode() != null) {

12
script/src/main/java/io/kestra/plugin/scripts/runner/docker/DockerService.java
View File

@@ -89,27 +89,27 @@ public class DockerService {
for (Credentials c : credentials) {
if (c.getUsername() != null) {
auths.put("username", runContext.render(c.getUsername()));
auths.put("username", runContext.render(c.getUsername()).as(String.class).orElse(null));
}
if (c.getPassword() != null) {
auths.put("password", runContext.render(c.getPassword()));
auths.put("password", runContext.render(c.getPassword()).as(String.class).orElse(null));
}
if (c.getRegistryToken() != null) {
auths.put("registrytoken", runContext.render(c.getRegistryToken()));
auths.put("registrytoken", runContext.render(c.getRegistryToken()).as(String.class).orElse(null));
}
if (c.getIdentityToken() != null) {
auths.put("identitytoken", runContext.render(c.getIdentityToken()));
auths.put("identitytoken", runContext.render(c.getIdentityToken()).as(String.class).orElse(null));
}
if (c.getAuth() != null) {
auths.put("auth", runContext.render(c.getAuth()));
auths.put("auth", runContext.render(c.getAuth()).as(String.class).orElse(null));
}
if (c.getRegistry() != null) {
registry = runContext.render(c.getRegistry());
registry = runContext.render(c.getRegistry()).as(String.class).orElse(null);
} else if (image != null) {
String renderedImage = runContext.render(image);
String detectedRegistry = registryUrlFromImage(renderedImage);

20
script/src/main/java/io/kestra/plugin/scripts/runner/docker/Memory.java
View File

@@ -1,6 +1,6 @@
package io.kestra.plugin.scripts.runner.docker;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.property.Property;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -25,8 +25,7 @@ public class Memory {
It is recommended that you allocate at least `6MB`."""
)
@PluginProperty(dynamic = true)
private String memory;
private Property<String> memory;
@Schema(
title = "The total amount of `memory` and `swap` that can be used by a container.",
@@ -34,24 +33,21 @@ public class Memory {
"using any swap. This is because `memorySwap` includes both the physical memory and swap space, " +
"while `memory` is only the amount of physical memory that can be used."
)
@PluginProperty(dynamic = true)
private String memorySwap;
private Property<String> memorySwap;
@Schema(
title = "A setting which controls the likelihood of the kernel to swap memory pages.",
description = "By default, the host kernel can swap out a percentage of anonymous pages used by a " +
"container. You can set `memorySwappiness` to a value between 0 and 100 to tune this percentage."
)
@PluginProperty(dynamic = true)
private String memorySwappiness;
private Property<String> memorySwappiness;
@Schema(
title = "Allows you to specify a soft limit smaller than `memory` which is activated when Docker detects contention or low memory on the host machine.",
description = "If you use `memoryReservation`, it must be set lower than `memory` for it to take precedence. " +
"Because it is a soft limit, it does not guarantee that the container doesn’t exceed the limit."
)
@PluginProperty(dynamic = true)
private String memoryReservation;
private Property<String> memoryReservation;
@Schema(
title = "The maximum amount of kernel memory the container can use.",
@@ -60,8 +56,7 @@ public class Memory {
"side effects on the host machine and on other containers. " +
"See the [kernel-memory docs](https://docs.docker.com/config/containers/resource_constraints/#--kernel-memory-details) for more details."
)
@PluginProperty(dynamic = true)
private String kernelMemory;
private Property<String> kernelMemory;
@Schema(
title = "By default, if an out-of-memory (OOM) error occurs, the kernel kills processes in a container.",
@@ -69,6 +64,5 @@ public class Memory {
"on containers where you have also set the `memory` option. If the `memory` flag is not set, the host " +
"can run out of memory, and the kernel may need to kill the host system’s processes to free the memory."
)
@PluginProperty
private Boolean oomKillDisable;
private Property<Boolean> oomKillDisable;
}