Compare commits

...

52 Commits

Author SHA1 Message Date
Florian Hussonnois
bb800948e5 chore: upgrade to version 'v0.17.6' 2024-06-25 16:42:44 +02:00
Florian Hussonnois
65f921d456 chore: add optional sequential id to executor
This commit adds an optional seqId property to the
Executor class that can be used to detect concurrent/stale updates
on execution.
2024-06-25 16:40:58 +02:00
YannC
ded21b0902 fix(jdbc): add timezone in JDBC url as it was using default JVM timezone (#4114) 2024-06-25 16:40:40 +02:00
Loïc Mathieu
7b109baac2 fix(core): the Request task can crash the worker
Fixes #4115
2024-06-25 16:40:28 +02:00
Loïc Mathieu
c11fe3466f fix(script): bad merge 2024-06-21 16:56:35 +02:00
Loïc Mathieu
94c5b7a6e4 chore: version 0.17.5 2024-06-21 16:54:16 +02:00
Loïc Mathieu
99ab5be8b9 fix(ui): amended editor content when switching from topology view (#4091) 2024-06-21 16:51:35 +02:00
Loïc Mathieu
aaa3a0ace0 fix(core): possible NPE on namespace files usage
Fixes #4078
2024-06-21 16:50:18 +02:00
brian.mulier
acc5a24d9a fix(core): add secret consumer when rendering variables for subflows
closes kestra-io/kestra-ee#1259
2024-06-20 15:04:32 +02:00
brian.mulier
892bb114ca chore: upgrade to version 'v0.17.4' 2024-06-17 12:41:03 +02:00
YannC
30e4fe4e0b fix(): namespaces files working in script tasks on Windows 2024-06-17 12:41:03 +02:00
brian.mulier
fcada08edd fix(core): nullable tenants & executions for execution skips 2024-06-17 12:32:01 +02:00
brian.mulier
a2df125a62 feat(*): skip executions for a tenant
part of kestra-io/kestra-ee#1247
2024-06-17 12:31:05 +02:00
brian.mulier
9d9c5dc1d1 fix(*): add tenant id to namespace identifier for skip execution by namespace
part of kestra-io/kestra-ee#1247
2024-06-17 12:30:55 +02:00
brian.mulier
dbb1a8eaa5 feat(*): skip executions for a namespace
part of kestra-io/kestra-ee#1247
2024-06-17 12:30:45 +02:00
YannC
1db6b57091 chore: upgrade to version 'v0.17.3' 2024-06-14 22:35:27 +02:00
Loïc Mathieu
934ea201a5 fix(webserver): add plugin alias icons
Fixes #4030
2024-06-14 22:21:29 +02:00
YannC
5dcd5b5af8 fix(ui): no stringify of json inputs (already string as coming form string input)
close #4033
2024-06-14 22:19:56 +02:00
YannC
000124f3dd fix(ui): only throw flow/execution not found on sse error when execution is not populated
close #4034
2024-06-14 22:19:53 +02:00
YannC
c37f104446 feat(ui): better handle of no permission (#4004) 2024-06-14 09:19:19 +02:00
yuri
3cfa48987f fix(ui): allow colon mark in label value (#4027) 2024-06-14 08:59:59 +02:00
Miloš Paunović
79c22ee22c chore(ui): only validate yaml files if they are flows (#4017) 2024-06-14 08:59:42 +02:00
Loïc Mathieu
d3a2fa13a5 fix(webserver): trim the flow when importing
Otherwhise the flows (except the first one) will have an empty line at the begining of the surce due to the way we split multiple flows.

Fixes #3915
2024-06-14 08:59:31 +02:00
Frank Tianyu Zeng
20078f1e19 fix(ui): improve the readability of error message in flows (#3901) 2024-06-14 08:59:19 +02:00
Miloš Paunović
72b86d9edf chore(ui): editor improvements (#4005)
* fix(ui): editor cursor position on windows is now recalculated properly

* fix(ui): binding file explorer context menu to id property instead of name
2024-06-14 08:58:57 +02:00
Miloš Paunović
59634133bc fix(ui): properly parsing json files in the editor (#4007) 2024-06-14 08:58:47 +02:00
Miloš Paunović
1e34a5528b chore(ui): added note for timezone settings (#3968) 2024-06-14 08:58:01 +02:00
Miloš Paunović
4aa3bd3ef2 chore(ui): added min and max values for int and float input types on flow execution (#3956) 2024-06-14 08:57:56 +02:00
YannC
f6581de304 fix(): replace Windows \ for / in LocalStorage 2024-06-14 08:57:42 +02:00
Florian Hussonnois
fd225d87b4 fix(core): properly inject pluginConfiguration for WorkingDirectory task (#4006)
fix: #4006
2024-06-14 08:57:36 +02:00
brian-mulier-p
9bb3f576ee fix(core): decrypt outputs for tasks within WorkingDirectory (#4001)
closes #4000
2024-06-14 08:57:28 +02:00
Florian Hussonnois
30cdb373cc fix(core): add unique prefix identifier for output files (#3991)
fix: #3991
2024-06-14 08:57:16 +02:00
YannC
59c7d6a567 chore: upgrade to version 'v0.17.2' 2024-06-10 16:24:37 +02:00
brian.mulier
9e4e5f891e fix(ui): namespace files calls were not including tenant 2024-06-10 16:23:42 +02:00
Milos Paunovic
ea3ba991d1 fix(ui): amended output preview for sqs trigger messages for ion files 2024-06-10 16:23:11 +02:00
Miloš Paunović
1024c77289 chore(ui): showing ee tooltip only on click (#3951) 2024-06-10 16:23:04 +02:00
Miloš Paunović
36b29d6065 chore(ui): showing ee tooltip on hover only once, then, just on click (#3944) 2024-06-10 16:22:57 +02:00
YannC
1c8177e185 chore: upgrade to version 0.17.1 2024-06-05 22:38:53 +02:00
brian.mulier
3dd5d6bb71 fix(ui): prevent the need of loading all flows for Flow tab to be displayed in editor 2024-06-05 22:38:53 +02:00
YannC
16a641693a fix(ui): avoid 404 with autocomplete when flow does not exist 2024-06-05 22:21:07 +02:00
YannC
efdb075155 fix(core): Now accept an extension for the file input
close #3858
2024-06-05 22:21:02 +02:00
Miloš Paunović
a99d52a406 fix(ui): added safety checks for all tour related calls (#3938) 2024-06-05 22:20:53 +02:00
YannC
852edea36e fix(ui): dont count flow in tutorial namespace 2024-06-05 22:20:45 +02:00
brian.mulier
defa426259 fix(ui): null-safe guided tour access in TriggerFlow.vue 2024-06-05 22:20:38 +02:00
Miloš Paunović
3aadcfd683 fix(ui): flow default inputs are now properly populated (#3934) 2024-06-05 22:20:30 +02:00
YannC
0f5d59103a fix(core): remove @NotEmpty
close #3920
2024-06-05 22:20:16 +02:00
YannC
50b9120434 fix(core): UploadFiles now handle subfolders 2024-06-05 22:19:53 +02:00
Anna Geller
896c761502 feat: switch from contact-us to demo 2024-06-05 22:19:39 +02:00
Loïc Mathieu
381d1b381f chore: fix docker image build 2024-06-04 15:29:51 +02:00
Loïc Mathieu
72a428a439 core: add default 'true' to docker task 2024-06-04 14:45:57 +02:00
Loïc Mathieu
7447e61dbc chore: fix docker workflow variable computation 2024-06-04 14:45:52 +02:00
Loïc Mathieu
45ffc3cc22 fix: Maven description 2024-06-04 11:11:48 +02:00
58 changed files with 518 additions and 160 deletions

View File

@@ -7,14 +7,7 @@ on:
description: 'Retag latest Docker images'
required: true
type: string
options:
- "true"
- "false"
skip-test:
description: 'Skip test'
required: false
type: string
default: "false"
default: "true"
options:
- "true"
- "false"
@@ -125,6 +118,16 @@ jobs:
packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
python-libs: kestra
steps:
- uses: actions/checkout@v4
# Vars
- name: Set image name
id: vars
run: |
TAG=${GITHUB_REF#refs/*/}
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
# Download release
- name: Download release
uses: robinraju/release-downloader@v1.10
@@ -137,14 +140,6 @@ jobs:
run: |
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
# Vars
- name: Set image name
id: vars
run: |
TAG=${GITHUB_REF#refs/*/}
echo "tag=${TAG}" >> $GITHUB_OUTPUT
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
# Docker setup
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
@@ -179,7 +174,7 @@ jobs:
- name: Retag to latest
if: github.event.inputs.retag-latest == 'true'
run: |
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest{1}', matrix.image.name) }}
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest{0}', matrix.image.name) }}
end:
runs-on: ubuntu-latest

View File

@@ -454,7 +454,7 @@ subprojects {
}
maven.pom {
description 'The modern, scalable orchestrator & scheduler open source platform'
description = 'The modern, scalable orchestrator & scheduler open source platform'
developers {
developer {

View File

@@ -37,6 +37,12 @@ public class ExecutorCommand extends AbstractServerCommand {
@CommandLine.Option(names = {"--skip-flows"}, split=",", description = "a list of flow identifiers (tenant|namespace|flowId) to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipFlows = Collections.emptyList();
@CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "a list of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipNamespaces = Collections.emptyList();
@CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "a list of tenants to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipTenants = Collections.emptyList();
@CommandLine.Option(names = {"--start-executors"}, split=",", description = "a list of Kafka Stream executors to start, separated by a command. Use it only with the Kafka queue, for debugging purpose.")
private List<String> startExecutors = Collections.emptyList();
@@ -54,6 +60,8 @@ public class ExecutorCommand extends AbstractServerCommand {
public Integer call() throws Exception {
this.skipExecutionService.setSkipExecutions(skipExecutions);
this.skipExecutionService.setSkipFlows(skipFlows);
this.skipExecutionService.setSkipNamespaces(skipNamespaces);
this.skipExecutionService.setSkipTenants(skipTenants);
this.startExecutorService.applyOptions(startExecutors, notStartExecutors);

View File

@@ -32,7 +32,7 @@ public class LocalCommand extends StandAloneCommand {
"kestra.queue.type", "h2",
"kestra.storage.type", "local",
"kestra.storage.local.base-path", data.toString(),
"datasources.h2.url", "jdbc:h2:file:" + data.resolve("database") + ";DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=30000",
"datasources.h2.url", "jdbc:h2:file:" + data.resolve("database") + ";TIME ZONE=UTC;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=30000",
"datasources.h2.username", "sa",
"datasources.h2.password", "",
"datasources.h2.driverClassName", "org.h2.Driver",

View File

@@ -49,6 +49,12 @@ public class StandAloneCommand extends AbstractServerCommand {
@CommandLine.Option(names = {"--skip-flows"}, split=",", description = "a list of flow identifiers (namespace.flowId) to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipFlows = Collections.emptyList();
@CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "a list of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipNamespaces = Collections.emptyList();
@CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "a list of tenants to skip, separated by a coma; for troubleshooting purpose only")
private List<String> skipTenants = Collections.emptyList();
@CommandLine.Option(names = {"--no-tutorials"}, description = "Flag to disable auto-loading of tutorial flows.")
boolean tutorialsDisabled = false;
@@ -74,6 +80,8 @@ public class StandAloneCommand extends AbstractServerCommand {
public Integer call() throws Exception {
this.skipExecutionService.setSkipExecutions(skipExecutions);
this.skipExecutionService.setSkipFlows(skipFlows);
this.skipExecutionService.setSkipNamespaces(skipNamespaces);
this.skipExecutionService.setSkipTenants(skipTenants);
this.startExecutorService.applyOptions(startExecutors, notStartExecutors);

View File

@@ -1,18 +1,21 @@
package io.kestra.core.models.flows.input;
import io.kestra.core.models.flows.Input;
import jakarta.validation.ConstraintViolationException;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.net.URI;
import jakarta.validation.ConstraintViolationException;
@SuperBuilder
@Getter
@NoArgsConstructor
public class FileInput extends Input<URI> {
@Builder.Default
public String extension = ".upl";
@Override
public void validate(URI input) throws ConstraintViolationException {
// no validation yet

View File

@@ -35,6 +35,11 @@ public class Executor {
private ExecutionResumed executionResumed;
private ExecutionResumed joinedExecutionResumed;
/**
* The sequence id should be incremented each time the execution is persisted after mutation.
*/
private long seqId = 0L;
/**
* List of {@link ExecutionKilled} to be propagated part of the execution.
*/
@@ -45,6 +50,12 @@ public class Executor {
this.offset = offset;
}
public Executor(Execution execution, Long offset, long seqId) {
this.execution = execution;
this.offset = offset;
this.seqId = seqId;
}
public Executor(WorkerTaskResult workerTaskResult) {
this.joinedWorkerTaskResult = workerTaskResult;
}
@@ -148,7 +159,18 @@ public class Executor {
public Executor serialize() {
return new Executor(
this.execution,
this.offset
this.offset,
this.seqId
);
}
/**
* Increments and returns the execution sequence id.
*
* @return the sequence id.
*/
public long incrementAndGetSeqId() {
this.seqId++;
return seqId;
}
}

View File

@@ -2,6 +2,7 @@ package io.kestra.core.runners;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.tasks.runners.PluginUtilsService;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.ListUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
@@ -85,10 +86,14 @@ public abstract class FilesService {
.filter(path -> pathMatcher.matches(runContext.tempDir().relativize(path)))
.map(throwFunction(path -> new AbstractMap.SimpleEntry<>(
runContext.tempDir().relativize(path).toString(),
runContext.storage().putFile(path.toFile())
runContext.storage().putFile(path.toFile(), resolveUniqueNameForFile(path))
)))
.toList()
.stream();
}
}
private static String resolveUniqueNameForFile(final Path path) {
return IdUtils.from(path.toString()) + "-" + path.toFile().getName();
}
}

View File

@@ -8,6 +8,7 @@ import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.Input;
import io.kestra.core.models.flows.Type;
import io.kestra.core.models.flows.input.ArrayInput;
import io.kestra.core.models.flows.input.FileInput;
import io.kestra.core.models.tasks.common.EncryptedString;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.storages.StorageInterface;
@@ -85,7 +86,9 @@ public class FlowInputOutput {
.subscribeOn(Schedulers.boundedElastic())
.map(throwFunction(input -> {
if (input instanceof CompletedFileUpload fileUpload) {
File tempFile = File.createTempFile(fileUpload.getFilename() + "_", ".upl");
String fileExtension = inputs.stream().filter(flowInput -> flowInput instanceof FileInput && flowInput.getId().equals(fileUpload.getFilename())).map(flowInput -> ((FileInput) flowInput).getExtension()).findFirst().orElse(".upl");
fileExtension = fileExtension.startsWith(".") ? fileExtension : "." + fileExtension;
File tempFile = File.createTempFile(fileUpload.getFilename() + "_", fileExtension);
try (var inputStream = fileUpload.getInputStream();
var outputStream = new FileOutputStream(tempFile)) {
long transferredBytes = inputStream.transferTo(outputStream);

View File

@@ -31,7 +31,7 @@ public class NamespaceFilesService {
private StorageInterface storageInterface;
public List<URI> inject(RunContext runContext, String tenantId, String namespace, Path basePath, NamespaceFiles namespaceFiles) throws Exception {
if (!namespaceFiles.getEnabled()) {
if (!Boolean.TRUE.equals(namespaceFiles.getEnabled())) {
return Collections.emptyList();
}

View File

@@ -339,7 +339,7 @@ public class RunContext {
if (execution.getTaskRunList() != null) {
Map<String, Object> outputs = new HashMap<>(execution.outputs());
if (decryptVariables) {
decryptOutputs(outputs);
outputs = decryptOutputs(outputs);
}
builder.put("outputs", outputs);
}
@@ -401,25 +401,37 @@ public class RunContext {
));
}
if (this.runContextLogger != null) {
builder.put("addSecretConsumer", (Consumer<String>) s -> this.runContextLogger.usedSecret(s));
}
return builder.build();
}
private void decryptOutputs(Map<String, Object> outputs) {
for (var entry: outputs.entrySet()) {
private Map<String, Object> decryptOutputs(Map<String, Object> mapToDecrypt) {
if (mapToDecrypt == null) {
return null;
}
Map<String, Object> decryptedMap = new HashMap<>();
for (var entry: mapToDecrypt.entrySet()) {
decryptedMap.put(entry.getKey(), entry.getValue());
if (entry.getValue() instanceof Map map) {
// if some outputs are of type EncryptedString we decode them and replace the object
// if some value are of type EncryptedString we decode them and replace the object
if (EncryptedString.TYPE.equalsIgnoreCase((String)map.get("type"))) {
try {
String decoded = decrypt((String) map.get("value"));
outputs.put(entry.getKey(), decoded);
decryptedMap.put(entry.getKey(), decoded);
} catch (GeneralSecurityException e) {
throw new RuntimeException(e);
}
} else {
decryptOutputs((Map<String, Object>) map);
decryptedMap.put(entry.getKey(), decryptOutputs((Map<String, Object>) map));
}
}
}
return decryptedMap;
}
private Map<String, Object> variables(TaskRun taskRun) {
@@ -502,6 +514,8 @@ public class RunContext {
runContext.runContextLogger = this.runContextLogger;
runContext.tempBasedPath = this.tempBasedPath;
runContext.temporaryDirectory = this.temporaryDirectory;
runContext.pluginConfiguration = this.pluginConfiguration;
runContext.secretKey = this.secretKey;
return runContext;
}
@@ -583,6 +597,17 @@ public class RunContext {
return newContext;
}
public RunContext forWorkingDirectoryTask(final Task task) {
Map<String, Object> decryptedVariables = new HashMap<>(this.variables);
if (this.variables.get("outputs") != null) {
decryptedVariables.put("outputs", decryptOutputs((Map<String, Object>) this.variables.get("outputs")));
}
RunContext newRunContext = this.clone(decryptedVariables);
newRunContext.initPluginConfiguration(applicationContext, task.getClass(), task.getType());
return newRunContext;
}
public RunContext forTaskRunner(TaskRunner taskRunner) {
this.initPluginConfiguration(applicationContext, taskRunner.getClass(), taskRunner.getType());

View File

@@ -285,7 +285,7 @@ public class Worker implements Service, Runnable, AutoCloseable {
WorkerTask currentWorkerTask = workingDirectory.workerTask(
workerTask.getTaskRun(),
currentTask,
runContext
runContext.forWorkingDirectoryTask(currentTask)
);
// all tasks will be handled immediately by the worker

View File

@@ -12,13 +12,23 @@ import java.util.List;
public class SkipExecutionService {
private volatile List<String> skipExecutions = Collections.emptyList();
private volatile List<FlowId> skipFlows = Collections.emptyList();
private volatile List<NamespaceId> skipNamespaces = Collections.emptyList();
private volatile List<String> skipTenants = Collections.emptyList();
public synchronized void setSkipExecutions(List<String> skipExecutions) {
this.skipExecutions = skipExecutions;
this.skipExecutions = skipExecutions == null ? Collections.emptyList() : skipExecutions;
}
public synchronized void setSkipFlows(List<String> skipFlows) {
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(flow -> FlowId.from(flow)).toList();
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).toList();
}
public synchronized void setSkipNamespaces(List<String> skipNamespaces) {
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).toList();
}
public synchronized void setSkipTenants(List<String> skipTenants) {
this.skipTenants = skipTenants == null ? Collections.emptyList() : skipTenants;
}
/**
@@ -38,17 +48,30 @@ public class SkipExecutionService {
@VisibleForTesting
boolean skipExecution(String tenant, String namespace, String flow, String executionId) {
return skipExecutions.contains(executionId) ||
skipFlows.contains(new FlowId(tenant, namespace, flow));
return (tenant != null && skipTenants.contains(tenant)) ||
skipNamespaces.contains(new NamespaceId(tenant, namespace)) ||
skipFlows.contains(new FlowId(tenant, namespace, flow)) ||
(executionId != null && skipExecutions.contains(executionId));
}
private static String[] splitIdParts(String id) {
return id.split("\\|");
}
record FlowId(String tenant, String namespace, String flow) {
static FlowId from(String flowId) {
String[] parts = flowId.split("\\|");
String[] parts = SkipExecutionService.splitIdParts(flowId);
if (parts.length == 3) {
return new FlowId(parts[0], parts[1], parts[2]);
}
return new FlowId(null, parts[0], parts[1]);
}
};
record NamespaceId(String tenant, String namespace) {
static NamespaceId from(String namespaceId) {
String[] parts = SkipExecutionService.splitIdParts(namespaceId);
return new NamespaceId(parts[0], parts[1]);
}
};
}

View File

@@ -270,7 +270,7 @@ public class WorkingDirectory extends Sequential implements NamespaceFilesInterf
}
}
if (this.namespaceFiles != null ) {
if (this.namespaceFiles != null && Boolean.TRUE.equals(this.namespaceFiles.getEnabled())) {
NamespaceFilesService namespaceFilesService = runContext.getApplicationContext().getBean(NamespaceFilesService.class);
namespaceFilesService.inject(runContext, taskRun.getTenantId(), taskRun.getNamespace(), runContext.tempDir(), this.namespaceFiles);
}

View File

@@ -21,6 +21,7 @@ import java.net.URI;
import java.security.GeneralSecurityException;
import java.util.List;
import java.util.Map;
import java.util.OptionalInt;
@SuperBuilder
@ToString
@@ -122,6 +123,16 @@ public class Request extends AbstractHttp implements RunnableTask<Request.Output
response = client
.toBlocking()
.exchange(request, Argument.STRING, Argument.STRING);
// check that the string is a valid Unicode string
if (response.getBody().isPresent()) {
OptionalInt illegalChar = response.body().chars().filter(c -> !Character.isDefined(c)).findFirst();
if (illegalChar.isPresent()) {
throw new IllegalArgumentException("Illegal unicode code point in request body: " + illegalChar.getAsInt() +
", the Request task only support valid Unicode strings as body.\n" +
"You can try using the Download task instead.");
}
}
} catch (HttpClientResponseException e) {
if (!allowFailed) {
throw e;

View File

@@ -80,7 +80,6 @@ public class DeleteFiles extends Task implements RunnableTask<DeleteFiles.Output
private String namespace;
@NotNull
@NotEmpty
@Schema(
title = "A file or a list of files from the given namespace.",
description = "String or a list of strings; each string can either be a regex glob pattern or a file path URI.",

View File

@@ -11,7 +11,6 @@ import io.kestra.core.runners.RunContext;
import io.kestra.core.services.FlowService;
import io.kestra.core.utils.Rethrow;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -84,7 +83,6 @@ public class DownloadFiles extends Task implements RunnableTask<DownloadFiles.Ou
private String namespace;
@NotNull
@NotEmpty
@Schema(
title = "A file or a list of files from the given namespace.",
description = "String or a list of strings; each string can either be a regex glob pattern or a file path URI.",
@@ -93,11 +91,19 @@ public class DownloadFiles extends Task implements RunnableTask<DownloadFiles.Ou
@PluginProperty(dynamic = true)
private Object files;
@Schema(
title = "The folder where the downloaded files will be stored"
)
@PluginProperty(dynamic = true)
@Builder.Default
private String destination = "";
@Override
public Output run(RunContext runContext) throws Exception {
Logger logger = runContext.logger();
String renderedNamespace = runContext.render(namespace);
String renderedDestination = runContext.render(destination);
// Check if namespace is allowed
RunContext.FlowInfo flowInfo = runContext.flowInfo();
FlowService flowService = runContext.getApplicationContext().getBean(FlowService.class);
@@ -120,7 +126,7 @@ public class DownloadFiles extends Task implements RunnableTask<DownloadFiles.Ou
namespaceFilesService.recursiveList(flowInfo.tenantId(), renderedNamespace, null).forEach(Rethrow.throwConsumer(uri -> {
if (patterns.stream().anyMatch(p -> p.matches(Path.of(uri.getPath())))) {
try (InputStream inputStream = namespaceFilesService.content(flowInfo.tenantId(), renderedNamespace, uri)) {
downloaded.put(uri.getPath(), runContext.storage().putFile(inputStream, uri.getPath()));
downloaded.put(uri.getPath(), runContext.storage().putFile(inputStream, destination + uri.getPath()));
logger.debug(String.format("Downloaded %s", uri));
}
}

View File

@@ -138,10 +138,10 @@ public class UploadFiles extends Task implements RunnableTask<UploadFiles.Output
});
// check for file in current tempDir that match regexs
List<PathMatcher> patterns = regexs.stream().map(reg -> FileSystems.getDefault().getPathMatcher("glob:" + reg)).toList();
for (File file : Objects.requireNonNull(runContext.tempDir().toFile().listFiles())) {
List<PathMatcher> patterns = regexs.stream().map(reg -> FileSystems.getDefault().getPathMatcher("glob:" + runContext.tempDir().toString() + checkLeadingSlash(reg))).toList();
for (File file : Objects.requireNonNull(listFilesRecursively(runContext.tempDir().toFile()))) {
if (patterns.stream().anyMatch(p -> p.matches(Path.of(file.toURI().getPath())))) {
String newFilePath = buildPath(renderedDestination, file.getName());
String newFilePath = buildPath(renderedDestination, file.getPath().replace(runContext.tempDir().toString(), ""));
storeNewFile(logger, runContext, storageInterface, flowInfo.tenantId(), newFilePath, new FileInputStream(file));
}
}
@@ -199,6 +199,24 @@ public class UploadFiles extends Task implements RunnableTask<UploadFiles.Output
}
}
private List<File> listFilesRecursively(File directory) throws IOException {
List<File> files = new ArrayList<>();
if (directory == null || !directory.isDirectory()) {
return files; // Handle invalid directory or not a directory
}
for (File file : directory.listFiles()) {
if (file.isFile()) {
files.add(file);
} else {
// Recursively call for subdirectories
files.addAll(listFilesRecursively(file));
}
}
return files;
}
@Builder
@Getter
public static class Output implements io.kestra.core.models.tasks.Output {

View File

@@ -4,6 +4,7 @@ import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
@@ -16,6 +17,14 @@ class SkipExecutionServiceTest {
@Inject
private SkipExecutionService skipExecutionService;
@BeforeEach
void resetAll() {
skipExecutionService.setSkipExecutions(null);
skipExecutionService.setSkipFlows(null);
skipExecutionService.setSkipNamespaces(null);
skipExecutionService.setSkipTenants(null);
}
@Test
void skipExecutionByExecutionId() {
var executionToSkip = "aaabbbccc";
@@ -65,4 +74,25 @@ class SkipExecutionServiceTest {
assertThat(skipExecutionService.skipExecution(null, "namespace", "not_skipped", "random"), is(false));
assertThat(skipExecutionService.skipExecution("tenant", "namespace", "not_skipped", "random"), is(false));
}
@Test
void skipExecutionByNamespace() {
skipExecutionService.setSkipNamespaces(List.of("tenant|namespace"));
assertThat(skipExecutionService.skipExecution("tenant", "namespace", "someFlow", "someExecution"), is(true));
assertThat(skipExecutionService.skipExecution(null, "namespace", "someFlow", "someExecution"), is(false));
assertThat(skipExecutionService.skipExecution("anotherTenant", "namespace", "someFlow", "someExecution"), is(false));
assertThat(skipExecutionService.skipExecution("tenant", "namespace", "anotherFlow", "anotherExecution"), is(true));
assertThat(skipExecutionService.skipExecution("tenant", "other.namespace", "someFlow", "someExecution"), is(false));
}
@Test
void skipExecutionByTenantId() {
skipExecutionService.setSkipTenants(List.of("tenant"));
assertThat(skipExecutionService.skipExecution("tenant", "namespace", "someFlow", "someExecution"), is(true));
assertThat(skipExecutionService.skipExecution("anotherTenant", "namespace", "someFlow", "someExecution"), is(false));
assertThat(skipExecutionService.skipExecution("tenant", "another.namespace", "someFlow", "someExecution"), is(true));
assertThat(skipExecutionService.skipExecution("anotherTenant", "another.namespace", "someFlow", "someExecution"), is(false));
}
}

View File

@@ -6,22 +6,23 @@ import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.tasks.common.EncryptedString;
import io.kestra.core.runners.AbstractMemoryRunnerTest;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.runners.RunnerUtils;
import io.kestra.core.storages.InternalStorage;
import io.kestra.core.storages.StorageContext;
import io.kestra.core.storages.StorageInterface;
import io.kestra.core.utils.IdUtils;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.RetryingTest;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.security.GeneralSecurityException;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
@@ -30,7 +31,6 @@ import java.util.concurrent.TimeoutException;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.io.FileMatchers.anExistingFile;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -38,6 +38,9 @@ public class WorkingDirectoryTest extends AbstractMemoryRunnerTest {
@Inject
Suite suite;
@Inject
RunContextFactory runContextFactory;
@Test
void success() throws TimeoutException {
suite.success(runnerUtils);
@@ -83,6 +86,11 @@ public class WorkingDirectoryTest extends AbstractMemoryRunnerTest {
suite.outputFiles(runnerUtils);
}
@Test
void encryption() throws Exception {
suite.encryption(runnerUtils, runContextFactory);
}
@Singleton
public static class Suite {
@Inject
@@ -154,8 +162,15 @@ public class WorkingDirectoryTest extends AbstractMemoryRunnerTest {
storageContext
, storageInterface
);
URI fileURI = URI.create("kestra:" + storageContext.getContextStorageURI() + "/input.txt");
assertThat(new String(storage.getFile(fileURI).readAllBytes()), is("Hello World"));
TaskRun taskRun = execution.getTaskRunList().get(1);
Map<String, Object> outputs = taskRun.getOutputs();
assertThat(outputs, hasKey("uris"));
URI uri = URI.create(((Map<String, String>) outputs.get("uris")).get("input.txt"));
assertTrue(uri.toString().endsWith("input.txt"));
assertThat(new String(storage.getFile(uri).readAllBytes()), is("Hello World"));
}
@SuppressWarnings("unchecked")
@@ -236,6 +251,18 @@ public class WorkingDirectoryTest extends AbstractMemoryRunnerTest {
assertThat(execution.findTaskRunsByTaskId("t3").get(0).getOutputs().get("value"), is("third"));
}
public void encryption(RunnerUtils runnerUtils, RunContextFactory runContextFactory) throws TimeoutException, GeneralSecurityException {
Execution execution = runnerUtils.runOne(null, "io.kestra.tests", "working-directory-taskrun-encrypted");
assertThat(execution.getTaskRunList(), hasSize(3));
Map<String, Object> encryptedString = (Map<String, Object>) execution.findTaskRunsByTaskId("encrypted").get(0).getOutputs().get("value");
assertThat(encryptedString.get("type"), is(EncryptedString.TYPE));
String encryptedValue = (String) encryptedString.get("value");
assertThat(encryptedValue, is(not("Hello World")));
assertThat(runContextFactory.of().decrypt(encryptedValue), is("Hello World"));
assertThat(execution.findTaskRunsByTaskId("decrypted").get(0).getOutputs().get("value"), is("Hello World"));
}
private void put(String path, String content) throws IOException {
storageInterface.put(
null,

View File

@@ -0,0 +1,13 @@
id: working-directory-taskrun-encrypted
namespace: io.kestra.tests
tasks:
- id: workingDir
type: io.kestra.plugin.core.flow.WorkingDirectory
tasks:
- id: encrypted
type: io.kestra.core.tasks.test.Encrypted
format: "Hello World"
- id: decrypted
type: io.kestra.plugin.core.debug.Return
format: "{{outputs.encrypted.value}}"

View File

@@ -1,4 +1,4 @@
version=0.17.0
version=0.17.6
jacksonVersion=2.16.2
micronautVersion=4.4.3
@@ -7,4 +7,4 @@ slf4jVersion=2.0.13
org.gradle.parallel=true
org.gradle.caching=true
org.gradle.priority=low
org.gradle.priority=low

View File

@@ -1,6 +1,6 @@
datasources:
h2:
url: jdbc:h2:mem:public;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
url: jdbc:h2:mem:public;TIME ZONE=UTC;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
username: sa
password: ""
driverClassName: org.h2.Driver

View File

@@ -131,10 +131,10 @@ public class CommandsWrapper implements TaskCommands {
@SuppressWarnings("unchecked")
public ScriptOutput run() throws Exception {
List<String> filesToUpload = new ArrayList<>();
if (this.namespaceFiles != null) {
String tenantId = ((Map<String, String>) runContext.getVariables().get("flow")).get("tenantId");
String namespace = ((Map<String, String>) runContext.getVariables().get("flow")).get("namespace");
String tenantId = ((Map<String, String>) runContext.getVariables().get("flow")).get("tenantId");
String namespace = ((Map<String, String>) runContext.getVariables().get("flow")).get("namespace");
if (this.namespaceFiles != null && Boolean.TRUE.equals(this.namespaceFiles.getEnabled())) {
NamespaceFilesService namespaceFilesService = runContext.getApplicationContext().getBean(NamespaceFilesService.class);
List<URI> injectedFiles = namespaceFilesService.inject(
runContext,

View File

@@ -78,7 +78,7 @@ public class LocalStorage implements StorageInterface {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
uris.add(URI.create(file.toString()));
uris.add(URI.create(file.toString().replace("\\", "/")));
return FileVisitResult.CONTINUE;
}
@@ -90,7 +90,7 @@ public class LocalStorage implements StorageInterface {
}
});
URI fsPathUri = URI.create(fsPath.toString());
URI fsPathUri = URI.create(fsPath.toString().replace("\\", "/"));
return uris.stream().sorted(Comparator.reverseOrder())
.map(fsPathUri::relativize)
.map(URI::getPath)
@@ -115,7 +115,7 @@ public class LocalStorage implements StorageInterface {
URI relative = URI.create(
getPath(tenantId, null).relativize(
Path.of(file.toUri())
).toString()
).toString().replace("\\", "/")
);
return getAttributes(tenantId, relative);
}))

View File

@@ -1,6 +1,10 @@
<template>
<el-tooltip :persistent="false" :focus-on-show="true" popper-class="ee-tooltip" :disabled="!disabled" :placement="placement">
<el-tooltip :visible="visible" :persistent="false" :focus-on-show="true" popper-class="ee-tooltip" :disabled="!disabled" :placement="placement">
<template #content v-if="link">
<el-button circle class="ee-tooltip-close" @click="changeVisibility(false)">
<Close />
</el-button>
<p>{{ $t("ee-tooltip.features-blocked") }}</p>
<a
@@ -13,7 +17,7 @@
</a>
</template>
<template #default>
<span ref="slot-container">
<span ref="slot-container" class="cursor-pointer" @click="changeVisibility()">
<slot />
<lock v-if="disabled" />
</span>
@@ -22,10 +26,11 @@
</template>
<script>
import Close from "vue-material-design-icons/Close.vue";
import Lock from "vue-material-design-icons/Lock.vue";
export default {
components: {Lock},
components: {Close, Lock},
props: {
top: {
type: Boolean,
@@ -48,6 +53,16 @@
default: undefined
},
},
data() {
return {
visible: false,
}
},
methods: {
changeVisibility(visible = true) {
this.visible = visible
}
},
computed: {
link() {
@@ -83,5 +98,13 @@
:deep(.material-design-icon) > .material-design-icon__svg {
bottom: -0.125em;
}
.ee-tooltip-close {
position: absolute;
top: 0;
right: 0;
border: none;
margin: 0.5rem;
}
</style>

View File

@@ -69,6 +69,14 @@
if (oldValue.name === newValue.name && this.previousExecutionId !== this.$route.params.id) {
this.follow()
}
// if we change the execution id, we need to close the sse
if (this.$route.params.id != this.execution.id) {
this.closeSSE();
window.removeEventListener("popstate", this.follow)
this.$store.commit("execution/setExecution", undefined);
this.$store.commit("flow/setFlow", undefined);
this.$store.commit("flow/setFlowGraph", undefined);
}
},
},
methods: {
@@ -91,13 +99,16 @@
}
// sse.onerror doesnt return the details of the error
// but as our emitter can only throw an error on 404
// we can safely assume that the error
// we can safely assume that the error is a 404
// if execution is not defined
this.sse.onerror = () => {
this.$store.dispatch("core/showMessage", {
variant: "error",
title: this.$t("error"),
message: this.$t("errors.404.flow or execution"),
});
if (!this.execution) {
this.$store.dispatch("core/showMessage", {
variant: "error",
title: this.$t("error"),
message: this.$t("errors.404.flow or execution"),
});
}
}
});
},

View File

@@ -41,7 +41,7 @@
if (this.$route.query.reset) {
localStorage.setItem("tourDoneOrSkip", undefined);
this.$store.commit("core/setGuidedProperties", {tourStarted: false});
this.$tours["guidedTour"].start();
this.$tours["guidedTour"]?.start();
}
this.setupFlow()
},

View File

@@ -67,7 +67,7 @@
},
methods: {
stopTour() {
this.$tours["guidedTour"].stop();
this.$tours["guidedTour"]?.stop();
this.$store.commit("core/setGuidedProperties", {tourStarted: false});
},
},
@@ -79,7 +79,7 @@
if (!this.guidedProperties.tourStarted
&& localStorage.getItem("tourDoneOrSkip") !== "true"
&& this.total === 0) {
this.$tours["guidedTour"].start();
this.$tours["guidedTour"]?.start();
}
}, 200)
window.addEventListener("popstate", () => {

View File

@@ -55,7 +55,7 @@
handler: function (newValue) {
if (newValue?.manuallyContinue) {
setTimeout(() => {
this.$tours["guidedTour"].nextStep();
this.$tours["guidedTour"]?.nextStep();
this.$store.commit("core/setGuidedProperties", {manuallyContinue: false});
}, 500);
}

View File

@@ -91,12 +91,12 @@
},
methods: {
onClick() {
if (this.$tours["guidedTour"].isRunning.value) {
this.$tours["guidedTour"].nextStep();
if (this.$tours["guidedTour"]?.isRunning?.value) {
this.$tours["guidedTour"]?.nextStep();
this.$store.dispatch("api/events", {
type: "ONBOARDING",
onboarding: {
step: this.$tours["guidedTour"].currentStep._value,
step: this.$tours["guidedTour"]?.currentStep?._value,
action: "next",
template: this.guidedProperties.template
},
@@ -131,7 +131,7 @@
},
beforeClose(done){
if(this.guidedProperties.tourStarted) return;
this.reset();
done()
}

View File

@@ -185,9 +185,9 @@
padding: calc(2 * var(--spacer)) $spacer !important;
font-family: $font-family-monospace;
background-color: white;
white-space: pre;
white-space: normal;
border-top: 1px solid var(--bs-gray-300);
text-wrap: initial;
text-wrap: wrap;
html.dark & {
color: white;

View File

@@ -188,6 +188,11 @@
}
},
created() {
// Auth but no permission at all or no permission to load execution stats
if (this.user && (!this.user.hasAnyRole() || !this.user.hasAnyActionOnAnyNamespace(permission.EXECUTION, action.READ))) {
this.$router.push({name:"errors/403"});
return;
}
this.load();
},
watch: {
@@ -239,8 +244,10 @@
return _merge(base, queryFilter)
},
load() {
this.loadStats();
this.haveExecutions();
if (this.user && this.user.hasAnyActionOnAnyNamespace(permission.EXECUTION, action.READ)) {
this.loadStats();
this.haveExecutions();
}
},
haveExecutions() {
let params = {

View File

@@ -141,8 +141,8 @@
</template>
<template #default="{data, node}">
<el-dropdown
:ref="`dropdown__${data.fileName}`"
@contextmenu.prevent.stop="toggleDropdown(`dropdown__${data.fileName}`)"
:ref="`dropdown__${data.id}`"
@contextmenu.prevent.stop="toggleDropdown(`dropdown__${data.id}`)"
trigger="contextmenu"
class="w-100"
>
@@ -385,7 +385,7 @@
},
computed: {
...mapState({
flows: (state) => state.flow.flows,
flow: (state) => state.flow.flow,
explorerVisible: (state) => state.editor.explorerVisible,
}),
folders() {
@@ -765,17 +765,17 @@
(function pushItemToFolder(basePath = "", array) {
for (const item of array) {
const folderPath = `${basePath}${item.fileName}`;
if (folderPath === SELF.dialog.folder && Array.isArray(item.children)) {
item.children = SELF.sorted([...item.children, NEW]);
return true; // Return true if the folder is found and item is pushed
}
if (Array.isArray(item.children) && pushItemToFolder(`${folderPath}/`, item.children)) {
return true; // Return true if the folder is found and item is pushed in recursive call
}
}
return false;
})(undefined, this.items);
}
@@ -883,9 +883,9 @@
},
},
watch: {
flows: {
flow: {
handler(flow) {
if (flow && flow.length) {
if (flow) {
this.changeOpenedTabs({
action: "open",
name: "Flow",
@@ -948,21 +948,21 @@
.empty {
position: relative;
top: 100px;
text-align: center;
text-align: center;
color: white;
html.light & {
color: $tertiary;
}
& img {
margin-bottom: 2rem;
margin-bottom: 2rem;
}
& h3 {
font-size: var(--font-size-lg);
font-weight: 500;
margin-bottom: .5rem;
font-weight: 500;
margin-bottom: .5rem;
}
& p {

View File

@@ -114,6 +114,8 @@
},
});
const isCurrentTabFlow = computed(() => currentTab?.value?.extension === undefined)
const flowErrors = computed(() => {
const isFlow = currentTab?.value?.extension === undefined;
@@ -639,7 +641,6 @@
})
.then(() => {
overrideFlow.value = true;
console.log("pop");
return true;
})
.catch(() => {
@@ -989,7 +990,7 @@
@save="save"
@execute="execute"
v-model="flowYaml"
schema-type="flow"
:schema-type="isCurrentTabFlow? 'flow': undefined"
:lang="currentTab?.extension === undefined ? 'yaml' : undefined"
:extension="currentTab?.extension"
@update:model-value="editorUpdate"

View File

@@ -39,18 +39,26 @@
@update:model-value="onChange"
show-password
/>
<el-input-number
v-if="input.type === 'INT'"
v-model="inputs[input.id]"
@update:model-value="onChange"
:step="1"
/>
<el-input-number
v-if="input.type === 'FLOAT'"
v-model="inputs[input.id]"
@update:model-value="onChange"
:step="0.001"
/>
<span v-if="input.type === 'INT'">
<el-input-number
v-model="inputs[input.id]"
@update:model-value="onChange"
:min="input.min"
:max="input.max && input.max >= (input.min || -Infinity) ? input.max : Infinity"
:step="1"
/>
<div v-if="input.min || input.max" class="hint">{{ numberHint(input) }}</div>
</span>
<span v-if="input.type === 'FLOAT'">
<el-input-number
v-model="inputs[input.id]"
@update:model-value="onChange"
:min="input.min"
:max="input.max && input.max >= (input.min || -Infinity) ? input.max : Infinity"
:step="0.001"
/>
<div v-if="input.min || input.max" class="hint">{{ numberHint(input) }}</div>
</span>
<el-radio-group
v-if="input.type === 'BOOLEAN'"
v-model="inputs[input.id]"
@@ -182,6 +190,18 @@
this.inputs[input.id] = e.target.files[0];
this.onChange();
},
numberHint(input){
const {min, max} = input;
if (min !== undefined && max !== undefined) {
if(min > max) return `Minimum value ${min} is larger than maximum value ${max}, so we've removed the upper limit.`;
return `Minimum value is ${min}, maximum value is ${max}.`;
} else if (min !== undefined) {
return `Minimum value is ${min}.`;
} else if (max !== undefined) {
return `Maximum value is ${max}.`;
} else return false;
}
},
watch: {
inputs: {
@@ -199,5 +219,8 @@
</script>
<style scoped lang="scss">
.hint {
font-size: var(--font-size-xs);
color: var(--bs-gray-700);
}
</style>

View File

@@ -16,6 +16,7 @@
import JsonWorker from "monaco-editor/esm/vs/language/json/json.worker?worker";
import {configureMonacoYaml} from "monaco-yaml";
import {yamlSchemas} from "override/utils/yamlSchemas";
import {editorViewTypes} from "../../utils/constants";
import Utils from "../../utils/utils";
import YamlUtils from "../../utils/yamlUtils";
import uniqBy from "lodash/uniqBy";
@@ -56,7 +57,8 @@
...mapState({
currentTab: (state) => state.editor.current,
tabs: (state) => state.editor.tabs,
flow: (state) => state.flow.flow
flow: (state) => state.flow.flow,
view: (state) => state.editor.view
}),
prefix() {
return this.schemaType ? `${this.schemaType}-` : "";
@@ -430,7 +432,8 @@
id: subflowTask.flowId,
revision: subflowTask.revision,
source: false,
store: false
store: false,
deleted: true
}
)).inputs?.map(input => input.id) ?? [];
} catch (e) {
@@ -631,6 +634,8 @@
}
}
});
setTimeout(() => monaco.editor.remeasureFonts(), 1)
this.$emit("editorDidMount", this.editor);
},
async changeTab(pathOrName, valueSupplier, useModelCache = true) {
@@ -672,6 +677,8 @@
this.editor.focus();
},
destroy: function () {
if(this.view === editorViewTypes.TOPOLOGY) return;
this.subflowAutocompletionProvider?.dispose();
this.pebbleAutocompletion?.dispose();
this.nestedFieldAutocompletionProvider?.dispose();

View File

@@ -28,6 +28,8 @@
</script>
<script>
import {mapMutations} from "vuex";
export default {
props: {
type: {
@@ -37,7 +39,10 @@
},
emits: ["switch-view"],
methods: {
...mapMutations("editor", ["changeView"]),
switchView(view) {
this.changeView(view)
this.$emit("switch-view", view)
},
buttonType(view) {

View File

@@ -48,9 +48,13 @@
[]
)
.forEach(label => {
const split = label.split(":");
const separatorIndex = label.indexOf(":");
labels.set(split[0], split[1]);
if (separatorIndex === -1) {
return;
}
labels.set(label.slice(0, separatorIndex), label.slice(separatorIndex + 1));
})
return labels;

View File

@@ -69,7 +69,7 @@
<Slack class="align-middle" /> {{ $t("join community") }}
</a>
<a
href="https://kestra.io/contact-us?utm_source=app&utm_content=top-nav-bar"
href="https://kestra.io/demo?utm_source=app&utm_content=top-nav-bar"
target="_blank"
class="d-flex gap-2 el-dropdown-menu__item"
>
@@ -147,7 +147,7 @@
localStorage.setItem("tourDoneOrSkip", undefined);
this.$store.commit("core/setGuidedProperties", {tourStarted: false});
this.$tours["guidedTour"].start();
this.$tours["guidedTour"]?.start();
}
}
};

View File

@@ -21,6 +21,8 @@
<script>
import {mapState} from "vuex";
import _uniqBy from "lodash/uniqBy";
import permission from "../../models/permission";
import action from "../../models/action";
export default {
props: {
@@ -43,14 +45,17 @@
},
emits: ["update:modelValue"],
created() {
this.$store
.dispatch("namespace/loadNamespacesForDatatype", {dataType: this.dataType})
.then(() => {
this.groupedNamespaces = this.groupNamespaces(this.datatypeNamespaces);
});
if (this.user && this.user.hasAnyActionOnAnyNamespace(permission.NAMESPACE, action.READ)) {
this.$store
.dispatch("namespace/loadNamespacesForDatatype", {dataType: this.dataType})
.then(() => {
this.groupedNamespaces = this.groupNamespaces(this.datatypeNamespaces);
});
}
},
computed: {
...mapState("namespace", ["datatypeNamespaces"])
...mapState("namespace", ["datatypeNamespaces"]),
...mapState("auth", ["user"]),
},
data() {
return {

View File

@@ -5,7 +5,7 @@
<Block :heading="$t('settings.blocks.configuration.label')">
<template #content>
<Row>
<Column :label="$t('settings.blocks.localization.fields.language')">
<Column :label="$t('settings.blocks.configuration.fields.language')">
<el-select :model-value="lang" @update:model-value="onLang">
<el-option
v-for="item in langOptions"
@@ -124,7 +124,7 @@
</template>
</Block>
<Block :heading="$t('settings.blocks.localization.label')">
<Block :heading="$t('settings.blocks.localization.label')" :note="$t('settings.blocks.localization.note')">
<template #content>
<Row>
<Column :label="$t('settings.blocks.localization.fields.time_zone')">

View File

@@ -1,14 +1,24 @@
<template>
<section>
<h1 class="heading" v-text="heading" />
<h1 class="heading">
<el-popover v-if="note" :content="note" trigger="hover" :width="400" class="info">
<template #reference>
<InformationOutline />
</template>
</el-popover>
<span>{{ heading }}</span>
</h1>
<slot name="content" />
<el-divider v-if="!last" />
</section>
</template>
<script setup lang="ts">
import InformationOutline from "vue-material-design-icons/InformationOutline.vue";
defineProps({
heading: {type: String, required: true},
note: {type: String, default: undefined},
last: {type: Boolean, default: false},
});
</script>
@@ -20,9 +30,16 @@ section {
margin: calc($spacer * 2);
& > h1.heading {
display: flex;
align-items: center;
margin-bottom: calc($spacer * 2);
font-size: calc($font-size-base * 1.5);
font-weight: 600;
& > span.el-tooltip__trigger {
cursor: pointer;
margin-right: calc($spacer / 2);
}
}
}
</style>

View File

@@ -183,17 +183,17 @@ export default {
}
},
save() {
if (this.$tours["guidedTour"].isRunning.value && !this.guidedProperties.saveFlow) {
if (this.$tours["guidedTour"]?.isRunning?.value && !this.guidedProperties.saveFlow) {
this.$store.dispatch("api/events", {
type: "ONBOARDING",
onboarding: {
step: this.$tours["guidedTour"].currentStep._value,
step: this.$tours["guidedTour"]?.currentStep?._value,
action: "next",
template: this.guidedProperties.template
},
page: pageFromRoute(this.$router.currentRoute.value)
});
this.$tours["guidedTour"].nextStep();
this.$tours["guidedTour"]?.nextStep();
return;
}

View File

@@ -1,5 +1,6 @@
export default {
FLOW: "FLOW",
EXECUTION: "EXECUTION",
TEMPLATE: "TEMPLATE"
TEMPLATE: "TEMPLATE",
NAMESPACE: "NAMESPACE"
}

View File

@@ -23,6 +23,10 @@ class Me {
hasAnyActionOnAnyNamespace(permission, action) {
return true;
}
hasAnyRole() {
return true;
}
}
export default {

View File

@@ -6,6 +6,7 @@ export default {
explorerWidth: 20,
current: undefined,
tabs: [],
view: undefined,
},
mutations: {
updateOnboarding(state) {
@@ -84,5 +85,12 @@ export default {
state.tabs = [state.tabs[0]];
}
},
closeAllTabs(state) {
state.tabs = [];
state.current = undefined
},
changeView(state, view) {
state.view = view;
},
},
};

View File

@@ -205,7 +205,19 @@ export default {
return this.$http.get(`${apiUrl(this)}/executions/${options.executionId}/file/preview`, {
params: options
}).then(response => {
commit("setFilePreview", response.data)
let data = {...response.data}
// WORKAROUND, related to https://github.com/kestra-io/plugin-aws/issues/456
if(data.extension === "ion") {
const notObjects = data.content.some(e => typeof e !== "object");
if(notObjects) {
const content = data.content.length === 1 ? data.content[0] : data.content.join("\n");
data = {...data, type: "TEXT", content}
}
}
commit("setFilePreview", data)
})
},
setLabels(_, options) {

View File

@@ -37,7 +37,7 @@ export default {
}).then(response => {
commit("setFlows", response.data.results)
commit("setTotal", response.data.total)
commit("setOverallTotal", response.data.total)
commit("setOverallTotal", response.data.results.filter(f => f.namespace !== "tutorial").length)
return response.data;
})

View File

@@ -1,7 +1,9 @@
import Utils from "../utils/utils";
import {apiUrl} from "override/utils/route";
const BASE = (namespace) => `${apiUrl(this)}/namespaces/${namespace}`;
function base(namespace) {
return `${apiUrl(this)}/namespaces/${namespace}`;
}
const HEADERS = {headers: {"Content-Type": "multipart/form-data"}};
const slashPrefix = (path) => (path.startsWith("/") ? path : `/${path}`);
@@ -15,13 +17,13 @@ export default {
actions: {
// Create a directory
async createDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files/directory?path=${slashPrefix(payload.path)}`;
const URL = `${base.call(this, payload.namespace)}/files/directory?path=${slashPrefix(payload.path)}`;
await this.$http.post(URL);
},
// List directory content
async readDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files/directory${payload.path ? `?path=${slashPrefix(safePath(payload.path))}` : ""}`;
const URL = `${base.call(this, payload.namespace)}/files/directory${payload.path ? `?path=${slashPrefix(safePath(payload.path))}` : ""}`;
const request = await this.$http.get(URL);
return request.data ?? [];
@@ -33,21 +35,21 @@ export default {
const BLOB = new Blob([payload.content], {type: "text/plain"});
DATA.append("fileContent", BLOB);
const URL = `${BASE(payload.namespace)}/files?path=${slashPrefix(payload.path)}`;
const URL = `${base.call(this, payload.namespace)}/files?path=${slashPrefix(payload.path)}`;
await this.$http.post(URL, DATA, HEADERS);
},
// Get namespace file content
async readFile(_, payload) {
const URL = `${BASE(payload.namespace)}/files?path=${slashPrefix(safePath(payload.path))}`;
const request = await this.$http.get(URL);
const URL = `${base.call(this, payload.namespace)}/files?path=${slashPrefix(safePath(payload.path))}`;
const request = await this.$http.get(URL, {transformResponse: response => response, responseType: "json"})
return request.data ?? [];
},
// Search for namespace files
async searchFiles(_, payload) {
const URL = `${BASE(payload.namespace)}/files/search?q=${payload.query}`;
const URL = `${base.call(this, payload.namespace)}/files/search?q=${payload.query}`;
const request = await this.$http.get(URL);
return request.data ?? [];
@@ -59,31 +61,31 @@ export default {
const BLOB = new Blob([payload.content], {type: "text/plain"});
DATA.append("fileContent", BLOB);
const URL = `${BASE(payload.namespace)}/files?path=${slashPrefix(safePath(payload.path))}`;
const URL = `${base.call(this, payload.namespace)}/files?path=${slashPrefix(safePath(payload.path))}`;
await this.$http.post(URL, DATA, HEADERS);
},
// Move a file or directory
async moveFileDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files?from=${slashPrefix(payload.old)}&to=${slashPrefix(payload.new)}`;
const URL = `${base.call(this, payload.namespace)}/files?from=${slashPrefix(payload.old)}&to=${slashPrefix(payload.new)}`;
await this.$http.put(URL);
},
// Rename a file or directory
async renameFileDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files?from=${slashPrefix(payload.old)}&to=${slashPrefix(payload.new)}`;
const URL = `${base.call(this, payload.namespace)}/files?from=${slashPrefix(payload.old)}&to=${slashPrefix(payload.new)}`;
await this.$http.put(URL);
},
// Delete a file or directory
async deleteFileDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files?path=${slashPrefix(payload.path)}`;
const URL = `${base.call(this, payload.namespace)}/files?path=${slashPrefix(payload.path)}`;
await this.$http.delete(URL);
},
// Export namespace files as a ZIP
async exportFileDirectory(_, payload) {
const URL = `${BASE(payload.namespace)}/files/export`;
const URL = `${base.call(this, payload.namespace)}/files/export`;
const request = await this.$http.get(URL);
const name = payload.namespace + "_files.zip";

View File

@@ -310,6 +310,14 @@
"title": "Page not found",
"content": "The requested URL was not found on this server. <span class=\"text-muted\">Thats all we know.</span>",
"flow or execution": "The flow or execution you are looking for does not exist."
},
"401": {
"title": "Unauthorized",
"content": "You need to be authenticated to access this page."
},
"403": {
"title": "Access denied",
"content": "You don't have the required permissions to access this page."
}
},
"copy logs": "Copy logs",
@@ -546,7 +554,7 @@
"environment color setting": "Environment color",
"slack support": "Ask any question via Slack",
"join community": "Join the Community",
"reach us": "Reach out to us",
"reach us": "Talk to us",
"new version": "New version {version} available!",
"error detected": "Error(s) detected",
"warning detected": "Warning(s) detected",
@@ -709,6 +717,7 @@
"configuration": {
"label": "Main Configuration",
"fields": {
"language": "Language",
"default_namespace": "Default Namespace",
"log_level": "Default Log Level",
"log_display": "Default Log Display",
@@ -729,8 +738,8 @@
},
"localization": {
"label": "Date and Time Settings",
"note": "Note that this setting is used for displaying date and time properties in the UI. To schedule your flows in a timezone different than UTC, make sure to set the timezone property on the Schedule trigger in your flow code or your plugin defaults.",
"fields": {
"language": "Language",
"time_zone": "Time Zone",
"date_format": "Date Format"
}

View File

@@ -299,6 +299,14 @@
"title": "Page introuvable",
"content": "L'URL demandé est introuvable sur ce serveur. <span class=\"text-muted\">C'est tout ce que nous savons.</span>",
"flow or execution": "Le flow ou l'exécution demandé est introuvable."
},
"401": {
"title": "Non authentifié",
"content": "Vous devez être authentifié pour accéder à cette page."
},
"403": {
"title": "Accès refusé",
"content": "Vous n'avez pas les permissions suffisantes pour accéder à cette page."
}
},
"copy logs": "Copier les logs",
@@ -681,6 +689,7 @@
"configuration": {
"label": "Configuration Principale",
"fields": {
"language": "Langue",
"default_namespace": "Espace de noms par défaut",
"log_level": "Niveau d'affichage des journaux par défaut",
"log_display": "Affichage des journaux par défaut",
@@ -701,8 +710,8 @@
},
"localization": {
"label": "Paramètres de Date et d'Heure",
"note": "Remarque ce paramètre est utilisé pour afficher les propriétés de date et d'heure dans l'interface utilisateur. Pour planifier vos flux dans un fuseau horaire différent de l'UTC, assurez-vous de définir la propriété de fuseau horaire sur le déclencheur de planification dans le code de votre",
"fields": {
"language": "Langue",
"time_zone": "Fuseau Horaire",
"date_format": "Format de Date"
}

View File

@@ -10,10 +10,12 @@ export default class Inputs {
res = moment(res).toISOString()
} else if (type === "DURATION" || type === "TIME") {
res = moment().startOf("day").add(res, "seconds").toString()
} else if (type === "JSON" || type === "ARRAY") {
res = JSON.stringify(res).toString()
} else if (type === "ARRAY") {
res = JSON.stringify(res).toString();
} else if (type === "BOOLEAN" && type === undefined){
res = "undefined";
} else if (type === "STRING" && Array.isArray(res)){
res = res.toString();
}
return res;
}

View File

@@ -84,7 +84,7 @@ export const executeTask = (submitor, flow, values, options) => {
}
}
if(options.nextStep) submitor.$tours["guidedTour"].nextStep();
if(options.nextStep) submitor.$tours["guidedTour"]?.nextStep();
return response.data;
})

View File

@@ -274,7 +274,7 @@ public class FlowController {
namespace,
sources
.stream()
.map(flow -> FlowWithSource.of(yamlFlowParser.parse(flow, Flow.class), flow))
.map(flow -> FlowWithSource.of(yamlFlowParser.parse(flow, Flow.class), flow.trim()))
.toList(),
delete
);
@@ -727,7 +727,7 @@ public class FlowController {
if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) {
List<String> sources = List.of(new String(fileUpload.getBytes()).split("---"));
for (String source : sources) {
this.importFlow(tenantId, source);
this.importFlow(tenantId, source.trim());
}
} else if (fileName.endsWith(".zip")) {
try (ZipInputStream archive = new ZipInputStream(fileUpload.getInputStream())) {

View File

@@ -138,17 +138,13 @@ public class PluginController {
public MutableHttpResponse<Map<String, PluginIcon>> icons() {
Map<String, PluginIcon> icons = pluginRegistry.plugins()
.stream()
.flatMap(plugin -> Stream
.concat(
.flatMap(plugin -> Stream.of(
plugin.getTasks().stream(),
Stream.concat(
Stream.concat(
plugin.getTriggers().stream(),
plugin.getConditions().stream()
),
plugin.getTaskRunners().stream()
)
plugin.getTriggers().stream(),
plugin.getConditions().stream(),
plugin.getTaskRunners().stream()
)
.flatMap(i -> i)
.map(e -> new AbstractMap.SimpleEntry<>(
e.getName(),
new PluginIcon(
@@ -160,6 +156,20 @@ public class PluginController {
)
.filter(entry -> entry.getKey() != null)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a1, a2) -> a1));
// add aliases
Map<String, PluginIcon> aliasIcons = pluginRegistry.plugins().stream()
.flatMap(plugin -> plugin.getAliases().values().stream().map(e -> new AbstractMap.SimpleEntry<>(
e.getKey(),
new PluginIcon(
e.getKey().substring(e.getKey().lastIndexOf('.') + 1),
plugin.icon(e.getValue()),
FlowableTask.class.isAssignableFrom(e.getValue())
))))
.filter(entry -> entry.getKey() != null)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a1, a2) -> a1));
icons.putAll(aliasIcons);
return HttpResponse.ok(icons).header(HttpHeaders.CACHE_CONTROL, CACHE_DIRECTIVE);
}

View File

@@ -86,6 +86,8 @@ class PluginControllerTest {
);
assertThat(list.entrySet().stream().filter(e -> e.getKey().equals(Log.class.getName())).findFirst().orElseThrow().getValue().getIcon(), is(notNullValue()));
// test an alias
assertThat(list.entrySet().stream().filter(e -> e.getKey().equals("io.kestra.core.tasks.log.Log")).findFirst().orElseThrow().getValue().getIcon(), is(notNullValue()));
});
}

View File

@@ -109,7 +109,7 @@ kestra:
cls: io.kestra.core.runners.ExecutionQueued
datasources:
h2:
url: jdbc:h2:mem:public;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
url: jdbc:h2:mem:public;TIME ZONE=UTC;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
username: sa
password: ""
driverClassName: org.h2.Driver