chore(system): extract queue consumers processing into message handlers

When a flow have both a condition and a precondition, the condition was evaluated twice which lead to double execution triggered.

Fixes

.github/workflows/main-build.yml

@@ -51,7 +51,6 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
 
 
   publish-develop-maven:

.github/workflows/release-docker.yml

@@ -32,4 +32,3 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}

cli/src/main/java/io/kestra/cli/AbstractCommand.java

@@ -42,7 +42,7 @@ import picocli.CommandLine.Option;
 @Introspected
 public abstract class AbstractCommand implements Callable<Integer> {
     @Inject
-    protected ApplicationContext applicationContext;
+    private ApplicationContext applicationContext;
 
     @Inject
     private EndpointDefaultConfiguration endpointConfiguration;

cli/src/main/java/io/kestra/cli/App.java

@@ -7,7 +7,6 @@ import io.kestra.cli.commands.namespaces.NamespaceCommand;
 import io.kestra.cli.commands.plugins.PluginCommand;
 import io.kestra.cli.commands.servers.ServerCommand;
 import io.kestra.cli.commands.sys.SysCommand;
-import io.kestra.cli.commands.templates.TemplateCommand;
 import io.micronaut.configuration.picocli.MicronautFactory;
 import io.micronaut.configuration.picocli.PicocliRunner;
 import io.micronaut.context.ApplicationContext;
@@ -39,7 +38,6 @@ import java.util.concurrent.Callable;
         PluginCommand.class,
         ServerCommand.class,
         FlowCommand.class,
-        TemplateCommand.class,
         SysCommand.class,
         ConfigCommand.class,
         NamespaceCommand.class,
@@ -70,7 +68,7 @@ public class App implements Callable<Integer> {
         try {
              exitCode = new CommandLine(cls, new MicronautFactory(applicationContext)).execute(args);
         } catch (CommandLine.InitializationException e){
-            System.err.println("Could not initialize picocli CommandLine, err: " + e.getMessage());
+            System.err.println("Could not initialize picoli ComandLine, err: " + e.getMessage());
             e.printStackTrace();
             exitCode = 1;
         }

cli/src/main/java/io/kestra/cli/StandAloneRunner.java

@@ -4,6 +4,7 @@ import io.kestra.core.runners.*;
 import io.kestra.core.server.Service;
 import io.kestra.core.utils.Await;
 import io.kestra.core.utils.ExecutorsUtils;
+import io.kestra.executor.DefaultExecutor;
 import io.kestra.worker.DefaultWorker;
 import io.micronaut.context.ApplicationContext;
 import io.micronaut.context.annotation.Value;
@@ -49,7 +50,7 @@ public class StandAloneRunner implements Runnable, AutoCloseable {
         running.set(true);
 
         poolExecutor = executorsUtils.cachedThreadPool("standalone-runner");
-        poolExecutor.execute(applicationContext.getBean(ExecutorInterface.class));
+        poolExecutor.execute(applicationContext.getBean(DefaultExecutor.class));
 
         if (workerEnabled) {
             // FIXME: For backward-compatibility with Kestra 0.15.x and earliest we still used UUID for Worker ID instead of IdUtils

cli/src/main/java/io/kestra/cli/commands/flows/FlowCommand.java

@@ -19,8 +19,7 @@ import picocli.CommandLine;
         FlowDotCommand.class,
         FlowExportCommand.class,
         FlowUpdateCommand.class,
-        FlowUpdatesCommand.class,
-        FlowsSyncFromSourceCommand.class
+        FlowUpdatesCommand.class
     }
 )
 @Slf4j

cli/src/main/java/io/kestra/cli/commands/flows/FlowExpandCommand.java

@@ -1,36 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import io.kestra.cli.AbstractCommand;
-import io.kestra.core.models.flows.Flow;
-import io.kestra.core.models.validations.ModelValidator;
-import io.kestra.core.serializers.YamlParser;
-import jakarta.inject.Inject;
-import picocli.CommandLine;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-@CommandLine.Command(
-    name = "expand",
-    description = "Deprecated - expand a flow"
-)
-@Deprecated
-public class FlowExpandCommand extends AbstractCommand {
-
-    @CommandLine.Parameters(index = "0", description = "The flow file to expand")
-    private Path file;
-
-    @Inject
-    private ModelValidator modelValidator;
-
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-        stdErr("Warning, this functionality is deprecated and will be removed at some point.");
-        String content = IncludeHelperExpander.expand(Files.readString(file), file.getParent());
-        Flow flow = YamlParser.parse(content, Flow.class);
-        modelValidator.validate(flow);
-        stdOut(content);
-        return 0;
-    }
-}

cli/src/main/java/io/kestra/cli/commands/flows/FlowUpdatesCommand.java

@@ -21,6 +21,8 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 
+import static io.kestra.core.utils.Rethrow.throwFunction;
+
 @CommandLine.Command(
     name = "updates",
     description = "Create or update flows from a folder, and optionally delete the ones not present",
@@ -41,7 +43,6 @@ public class FlowUpdatesCommand extends AbstractApiCommand {
     @Inject
     private TenantIdSelectorService tenantIdSelectorService;
 
-    @SuppressWarnings("deprecation")
     @Override
     public Integer call() throws Exception {
         super.call();
@@ -50,13 +51,7 @@ public class FlowUpdatesCommand extends AbstractApiCommand {
             List<String> flows = files
                 .filter(Files::isRegularFile)
                 .filter(YamlParser::isValidExtension)
-                .map(path -> {
-                    try {
-                        return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                })
+                .map(throwFunction(path -> Files.readString(path, Charset.defaultCharset())))
                 .toList();
 
             String body = "";

cli/src/main/java/io/kestra/cli/commands/flows/FlowsSyncFromSourceCommand.java

@@ -1,55 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import io.kestra.cli.AbstractApiCommand;
-import io.kestra.cli.services.TenantIdSelectorService;
-import io.kestra.core.models.flows.FlowWithSource;
-import io.kestra.core.models.flows.GenericFlow;
-import io.kestra.core.repositories.FlowRepositoryInterface;
-import jakarta.inject.Inject;
-import java.util.List;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-@CommandLine.Command(
-    name = "syncFromSource",
-    description = "Update a single flow",
-    mixinStandardHelpOptions = true
-)
-@Slf4j
-public class FlowsSyncFromSourceCommand extends AbstractApiCommand {
-
-    @Inject
-    private TenantIdSelectorService tenantService;
-
-    @SuppressWarnings("deprecation")
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        FlowRepositoryInterface repository = applicationContext.getBean(FlowRepositoryInterface.class);
-        String tenant = tenantService.getTenantId(tenantId);
-
-        List<FlowWithSource> persistedFlows = repository.findAllWithSource(tenant);
-
-        int count = 0;
-        for (FlowWithSource persistedFlow : persistedFlows) {
-            // Ensure exactly one trailing newline. We need this new line
-            // because when we update a flow from its source,
-            // we don't update it if no change is detected.
-            // The goal here is to force an update from the source for every flows
-            GenericFlow flow = GenericFlow.fromYaml(tenant,persistedFlow.getSource() + System.lineSeparator());
-            repository.update(flow, persistedFlow);
-            stdOut("- %s.%s".formatted(flow.getNamespace(), flow.getId()));
-            count++;
-        }
-        stdOut("%s flow(s) successfully updated!".formatted(count));
-
-        return 0;
-    }
-
-    protected boolean loadExternalPlugins() {
-        return true;
-    }
-
-
-}

cli/src/main/java/io/kestra/cli/commands/flows/IncludeHelperExpander.java

@@ -1,40 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import com.google.common.io.Files;
-import lombok.SneakyThrows;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.stream.Collectors;
-
-@Deprecated
-public abstract class IncludeHelperExpander {
-
-    public static String expand(String value, Path directory) throws IOException {
-        return value.lines()
-            .map(line -> line.contains("[[>") && line.contains("]]") ? expandLine(line, directory) : line)
-            .collect(Collectors.joining("\n"));
-    }
-
-    @SneakyThrows
-    private static String expandLine(String line, Path directory) {
-        String prefix = line.substring(0, line.indexOf("[[>"));
-        String suffix = line.substring(line.indexOf("]]") + 2, line.length());
-        String file = line.substring(line.indexOf("[[>") + 3 , line.indexOf("]]")).strip();
-        Path includePath = directory.resolve(file);
-        List<String> include = Files.readLines(includePath.toFile(), Charset.defaultCharset());
-
-        // handle single line directly with the suffix (should be between quotes or double-quotes
-        if(include.size() == 1) {
-            String singleInclude = include.getFirst();
-            return prefix + singleInclude + suffix;
-        }
-
-        // multi-line will be expanded with the prefix but no suffix
-        return include.stream()
-            .map(includeLine -> prefix + includeLine)
-            .collect(Collectors.joining("\n"));
-    }
-}

cli/src/main/java/io/kestra/cli/commands/flows/namespaces/FlowNamespaceUpdateCommand.java

@@ -2,7 +2,6 @@ package io.kestra.cli.commands.flows.namespaces;
 
 import io.kestra.cli.AbstractValidateCommand;
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
-import io.kestra.cli.commands.flows.IncludeHelperExpander;
 import io.kestra.cli.services.TenantIdSelectorService;
 import io.kestra.core.serializers.YamlParser;
 import io.micronaut.core.type.Argument;
@@ -21,6 +20,8 @@ import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.util.List;
 
+import static io.kestra.core.utils.Rethrow.throwFunction;
+
 @CommandLine.Command(
     name = "update",
     description = "Update flows in namespace",
@@ -44,13 +45,7 @@ public class FlowNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCo
             List<String> flows = files
                 .filter(Files::isRegularFile)
                 .filter(YamlParser::isValidExtension)
-                .map(path -> {
-                    try {
-                        return IncludeHelperExpander.expand(Files.readString(path, Charset.defaultCharset()), path.getParent());
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                })
+                .map(throwFunction(path -> Files.readString(path, Charset.defaultCharset())))
                 .toList();
 
             String body = "";

cli/src/main/java/io/kestra/cli/commands/migrations/metadata/KvMetadataMigrationCommand.java

@@ -2,7 +2,6 @@ package io.kestra.cli.commands.migrations.metadata;
 
 import io.kestra.cli.AbstractCommand;
 import jakarta.inject.Inject;
-import jakarta.inject.Provider;
 import lombok.extern.slf4j.Slf4j;
 import picocli.CommandLine;
 
@@ -13,13 +12,13 @@ import picocli.CommandLine;
 @Slf4j
 public class KvMetadataMigrationCommand extends AbstractCommand {
     @Inject
-    private Provider<MetadataMigrationService> metadataMigrationServiceProvider;
+    private MetadataMigrationService metadataMigrationService;
 
     @Override
     public Integer call() throws Exception {
         super.call();
         try {
-            metadataMigrationServiceProvider.get().kvMigration();
+            metadataMigrationService.kvMigration();
         } catch (Exception e) {
             System.err.println("❌ KV Metadata migration failed: " + e.getMessage());
             e.printStackTrace();

cli/src/main/java/io/kestra/cli/commands/migrations/metadata/MetadataMigrationService.java

@@ -1,6 +1,5 @@
 package io.kestra.cli.commands.migrations.metadata;
 
-import com.google.common.annotations.VisibleForTesting;
 import io.kestra.core.models.kv.PersistedKvMetadata;
 import io.kestra.core.repositories.FlowRepositoryInterface;
 import io.kestra.core.repositories.KvMetadataRepositoryInterface;
@@ -10,37 +9,39 @@ import io.kestra.core.storages.StorageInterface;
 import io.kestra.core.storages.kv.InternalKVStore;
 import io.kestra.core.storages.kv.KVEntry;
 import io.kestra.core.tenant.TenantService;
-import io.kestra.core.utils.NamespaceUtils;
+import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
-import lombok.AllArgsConstructor;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.time.Instant;
-import java.util.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static io.kestra.core.utils.Rethrow.throwConsumer;
 import static io.kestra.core.utils.Rethrow.throwFunction;
 
 @Singleton
-@AllArgsConstructor
 public class MetadataMigrationService {
-    protected FlowRepositoryInterface flowRepository;
-    protected TenantService tenantService;
-    protected KvMetadataRepositoryInterface kvMetadataRepository;
-    protected StorageInterface storageInterface;
-    protected NamespaceUtils namespaceUtils;
+    @Inject
+    private TenantService tenantService;
 
-    @VisibleForTesting
-    public Map<String, List<String>> namespacesPerTenant() {
+    @Inject
+    private FlowRepositoryInterface flowRepository;
+
+    @Inject
+    private KvMetadataRepositoryInterface kvMetadataRepository;
+
+    @Inject
+    private StorageInterface storageInterface;
+
+    protected Map<String, List<String>> namespacesPerTenant() {
         String tenantId = tenantService.resolveTenant();
-        return Map.of(tenantId, Stream.concat(
-            Stream.of(namespaceUtils.getSystemFlowNamespace()),
-            flowRepository.findDistinctNamespace(tenantId).stream()
-        ).map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList());
+        return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId));
     }
 
     public void kvMigration() throws IOException {

cli/src/main/java/io/kestra/cli/commands/migrations/metadata/SecretsMetadataMigrationCommand.java

@@ -2,7 +2,6 @@ package io.kestra.cli.commands.migrations.metadata;
 
 import io.kestra.cli.AbstractCommand;
 import jakarta.inject.Inject;
-import jakarta.inject.Provider;
 import lombok.extern.slf4j.Slf4j;
 import picocli.CommandLine;
 
@@ -13,13 +12,13 @@ import picocli.CommandLine;
 @Slf4j
 public class SecretsMetadataMigrationCommand extends AbstractCommand {
     @Inject
-    private Provider<MetadataMigrationService> metadataMigrationServiceProvider;
+    private MetadataMigrationService metadataMigrationService;
 
     @Override
     public Integer call() throws Exception {
         super.call();
         try {
-            metadataMigrationServiceProvider.get().secretMigration();
+            metadataMigrationService.secretMigration();
         } catch (Exception e) {
             System.err.println("❌ Secrets Metadata migration failed: " + e.getMessage());
             e.printStackTrace();

cli/src/main/java/io/kestra/cli/commands/servers/ExecutorCommand.java

@@ -1,10 +1,8 @@
 package io.kestra.cli.commands.servers;
 
 import com.google.common.collect.ImmutableMap;
-import io.kestra.cli.services.TenantIdSelectorService;
 import io.kestra.core.models.ServerType;
-import io.kestra.core.repositories.LocalFlowRepositoryLoader;
-import io.kestra.core.runners.ExecutorInterface;
+import io.kestra.core.runners.Executor;
 import io.kestra.core.services.SkipExecutionService;
 import io.kestra.core.services.StartExecutorService;
 import io.kestra.core.utils.Await;
@@ -12,8 +10,6 @@ import io.micronaut.context.ApplicationContext;
 import jakarta.inject.Inject;
 import picocli.CommandLine;
 
-import java.io.File;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -23,9 +19,6 @@ import java.util.Map;
     description = "Start the Kestra executor"
 )
 public class ExecutorCommand extends AbstractServerCommand {
-    @CommandLine.Spec
-    CommandLine.Model.CommandSpec spec;
-
     @Inject
     private ApplicationContext applicationContext;
 
@@ -35,28 +28,22 @@ public class ExecutorCommand extends AbstractServerCommand {
     @Inject
     private StartExecutorService startExecutorService;
 
-    @CommandLine.Option(names = {"-f", "--flow-path"}, description = "Tenant identifier required to load flows from the specified path")
-    private File flowPath;
-
-    @CommandLine.Option(names = "--tenant", description = "Tenant identifier, Required to load flows from path")
-    private String tenantId;
-
-    @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "List of execution IDs to skip, separated by commas; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "The list of execution identifiers to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipExecutions = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-flows"}, split=",", description = "List of flow identifiers (tenant|namespace|flowId) to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-flows"}, split=",", description = "The list of flow identifiers (tenant|namespace|flowId) to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipFlows = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "List of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "The list of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipNamespaces = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "List of tenants to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "The list of tenants to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipTenants = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--start-executors"}, split=",", description = "List of Kafka Stream executors to start, separated by a command. Use it only with the Kafka queue; for debugging only")
+    @CommandLine.Option(names = {"--start-executors"}, split=",", description = "The list of Kafka Stream executors to start, separated by a command. Use it only with the Kafka queue, for debugging purpose.")
     private List<String> startExecutors = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--not-start-executors"}, split=",", description = "Lst of Kafka Stream executors to not start, separated by a command. Use it only with the Kafka queue; for debugging only")
+    @CommandLine.Option(names = {"--not-start-executors"}, split=",", description = "The list of Kafka Stream executors to not start, separated by a command. Use it only with the Kafka queue, for debugging purpose.")
     private List<String> notStartExecutors = Collections.emptyList();
 
     @SuppressWarnings("unused")
@@ -77,17 +64,7 @@ public class ExecutorCommand extends AbstractServerCommand {
 
         super.call();
 
-        if (flowPath != null) {
-            try {
-                LocalFlowRepositoryLoader localFlowRepositoryLoader = applicationContext.getBean(LocalFlowRepositoryLoader.class);
-                TenantIdSelectorService tenantIdSelectorService = applicationContext.getBean(TenantIdSelectorService.class);
-                localFlowRepositoryLoader.load(tenantIdSelectorService.getTenantId(this.tenantId), this.flowPath);
-            } catch (IOException e) {
-                throw new CommandLine.ParameterException(this.spec.commandLine(), "Invalid flow path", e);
-            }
-        }
-
-        ExecutorInterface executorService = applicationContext.getBean(ExecutorInterface.class);
+        Executor executorService = applicationContext.getBean(Executor.class);
         executorService.run();
 
         Await.until(() -> !this.applicationContext.isRunning());

cli/src/main/java/io/kestra/cli/commands/servers/IndexerCommand.java

@@ -23,7 +23,7 @@ public class IndexerCommand extends AbstractServerCommand {
     @Inject
     private SkipExecutionService skipExecutionService;
 
-    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
     private List<String> skipIndexerRecords = Collections.emptyList();
 
     @SuppressWarnings("unused")

cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java

@@ -42,7 +42,7 @@ public class StandAloneCommand extends AbstractServerCommand {
     @Nullable
     private FileChangedEventListener fileWatcher;
 
-    @CommandLine.Option(names = {"-f", "--flow-path"}, description = "Tenant identifier required to load flows from the specified path")
+    @CommandLine.Option(names = {"-f", "--flow-path"}, description = "the flow path containing flow to inject at startup (when running with a memory flow repository)")
     private File flowPath;
 
     @CommandLine.Option(names = "--tenant", description = "Tenant identifier, Required to load flows from path with the enterprise edition")
@@ -51,19 +51,19 @@ public class StandAloneCommand extends AbstractServerCommand {
     @CommandLine.Option(names = {"--worker-thread"}, description = "the number of worker threads, defaults to eight times the number of available processors. Set it to 0 to avoid starting a worker.")
     private int workerThread = defaultWorkerThread();
 
-    @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "a list of execution identifiers to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-executions"}, split=",", description = "a list of execution identifiers to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipExecutions = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-flows"}, split=",", description = "a list of flow identifiers (namespace.flowId) to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-flows"}, split=",", description = "a list of flow identifiers (namespace.flowId) to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipFlows = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "a list of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-namespaces"}, split=",", description = "a list of namespace identifiers (tenant|namespace) to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipNamespaces = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "a list of tenants to skip, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "a list of tenants to skip, separated by a coma; for troubleshooting purpose only")
     private List<String> skipTenants = Collections.emptyList();
 
-    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
     private List<String> skipIndexerRecords = Collections.emptyList();
 
     @CommandLine.Option(names = {"--no-tutorials"}, description = "Flag to disable auto-loading of tutorial flows.")

cli/src/main/java/io/kestra/cli/commands/servers/WebServerCommand.java

@@ -40,7 +40,7 @@ public class WebServerCommand extends AbstractServerCommand {
     @Option(names = {"--no-indexer"}, description = "Flag to disable starting an embedded indexer.")
     private boolean indexerDisabled = false;
 
-    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting only")
+    @CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
     private List<String> skipIndexerRecords = Collections.emptyList();
 
     @Override

cli/src/main/java/io/kestra/cli/commands/sys/SubmitQueuedCommand.java

@@ -7,7 +7,7 @@ import io.kestra.core.queues.QueueFactoryInterface;
 import io.kestra.core.queues.QueueInterface;
 import io.kestra.core.runners.ExecutionQueued;
 import io.kestra.core.services.ConcurrencyLimitService;
-import io.kestra.jdbc.runner.AbstractJdbcExecutionQueuedStorage;
+import io.kestra.jdbc.runner.AbstractJdbcExecutionQueuedStateStore;
 import io.micronaut.context.ApplicationContext;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
@@ -47,7 +47,7 @@ public class SubmitQueuedCommand extends AbstractCommand {
             return 1;
         }
         else if (queueType.get().equals("postgres") || queueType.get().equals("mysql") || queueType.get().equals("h2")) {
-            var executionQueuedStorage = applicationContext.getBean(AbstractJdbcExecutionQueuedStorage.class);
+            var executionQueuedStorage = applicationContext.getBean(AbstractJdbcExecutionQueuedStateStore.class);
             var concurrencyLimitService = applicationContext.getBean(ConcurrencyLimitService.class);
 
             for (ExecutionQueued queued : executionQueuedStorage.getAllForAllTenants()) {

cli/src/main/java/io/kestra/cli/commands/sys/SysCommand.java

@@ -1,7 +1,6 @@
 package io.kestra.cli.commands.sys;
 
 import io.kestra.cli.commands.sys.database.DatabaseCommand;
-import io.kestra.cli.commands.sys.statestore.StateStoreCommand;
 import io.micronaut.configuration.picocli.PicocliRunner;
 import lombok.extern.slf4j.Slf4j;
 import io.kestra.cli.AbstractCommand;
@@ -16,7 +15,6 @@ import picocli.CommandLine;
         ReindexCommand.class,
         DatabaseCommand.class,
         SubmitQueuedCommand.class,
-        StateStoreCommand.class
     }
 )
 @Slf4j

cli/src/main/java/io/kestra/cli/commands/sys/statestore/StateStoreCommand.java

@@ -1,27 +0,0 @@
-package io.kestra.cli.commands.sys.statestore;
-
-import io.kestra.cli.AbstractCommand;
-import io.kestra.cli.App;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import lombok.SneakyThrows;
-import picocli.CommandLine;
-
-@CommandLine.Command(
-    name = "state-store",
-    description = "Manage Kestra State Store",
-    mixinStandardHelpOptions = true,
-    subcommands = {
-        StateStoreMigrateCommand.class,
-    }
-)
-public class StateStoreCommand extends AbstractCommand {
-    @SneakyThrows
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        PicocliRunner.call(App.class, "sys", "state-store", "--help");
-
-        return 0;
-    }
-}

cli/src/main/java/io/kestra/cli/commands/sys/statestore/StateStoreMigrateCommand.java

@@ -1,81 +0,0 @@
-package io.kestra.cli.commands.sys.statestore;
-
-import io.kestra.cli.AbstractCommand;
-import io.kestra.core.models.flows.Flow;
-import io.kestra.core.repositories.FlowRepositoryInterface;
-import io.kestra.core.runners.RunContext;
-import io.kestra.core.runners.RunContextFactory;
-import io.kestra.core.storages.StateStore;
-import io.kestra.core.storages.StorageInterface;
-import io.kestra.core.utils.Slugify;
-import io.micronaut.context.ApplicationContext;
-import jakarta.inject.Inject;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-@CommandLine.Command(
-    name = "migrate",
-    description = "Migrate old state store files to use the new KV Store implementation.",
-    mixinStandardHelpOptions = true
-)
-@Slf4j
-public class StateStoreMigrateCommand extends AbstractCommand {
-    @Inject
-    private ApplicationContext applicationContext;
-
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        FlowRepositoryInterface flowRepository = this.applicationContext.getBean(FlowRepositoryInterface.class);
-        StorageInterface storageInterface = this.applicationContext.getBean(StorageInterface.class);
-        RunContextFactory runContextFactory = this.applicationContext.getBean(RunContextFactory.class);
-
-        flowRepository.findAllForAllTenants().stream().map(flow -> Map.entry(flow, List.of(
-            URI.create("/" + flow.getNamespace().replace(".", "/") + "/" + Slugify.of(flow.getId()) + "/states"),
-            URI.create("/" + flow.getNamespace().replace(".", "/") + "/states")
-        ))).map(potentialStateStoreUrisForAFlow -> Map.entry(potentialStateStoreUrisForAFlow.getKey(), potentialStateStoreUrisForAFlow.getValue().stream().flatMap(uri -> {
-            try {
-                return storageInterface.allByPrefix(potentialStateStoreUrisForAFlow.getKey().getTenantId(), potentialStateStoreUrisForAFlow.getKey().getNamespace(), uri, false).stream();
-            } catch (IOException e) {
-                return Stream.empty();
-            }
-        }).toList())).forEach(stateStoreFileUrisForAFlow -> stateStoreFileUrisForAFlow.getValue().forEach(stateStoreFileUri -> {
-            Flow flow = stateStoreFileUrisForAFlow.getKey();
-            String[] flowQualifierWithStateQualifiers = stateStoreFileUri.getPath().split("/states/");
-            String[] statesUriPart = flowQualifierWithStateQualifiers[1].split("/");
-
-            String stateName = statesUriPart[0];
-            String taskRunValue = statesUriPart.length > 2 ? statesUriPart[1] : null;
-            String stateSubName = statesUriPart[statesUriPart.length - 1];
-            boolean flowScoped = flowQualifierWithStateQualifiers[0].endsWith("/" + flow.getId());
-            StateStore stateStore = new StateStore(runContext(runContextFactory, flow), false);
-
-            try (InputStream is = storageInterface.get(flow.getTenantId(), flow.getNamespace(), stateStoreFileUri)) {
-                stateStore.putState(flowScoped, stateName, stateSubName, taskRunValue, is.readAllBytes());
-                storageInterface.delete(flow.getTenantId(), flow.getNamespace(), stateStoreFileUri);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }));
-
-        stdOut("Successfully ran the state-store migration.");
-        return 0;
-    }
-
-    private RunContext runContext(RunContextFactory runContextFactory, Flow flow) {
-        Map<String, String> flowVariables = new HashMap<>();
-        flowVariables.put("tenantId", flow.getTenantId());
-        flowVariables.put("id", flow.getId());
-        flowVariables.put("namespace", flow.getNamespace());
-        return runContextFactory.of(flow, Map.of("flow", flowVariables));
-    }
-}

cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java

@@ -1,34 +0,0 @@
-package io.kestra.cli.commands.templates;
-
-import io.kestra.cli.AbstractCommand;
-import io.kestra.cli.App;
-import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceCommand;
-import io.kestra.core.models.templates.TemplateEnabled;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-@CommandLine.Command(
-    name = "template",
-    description = "Manage templates",
-    mixinStandardHelpOptions = true,
-    subcommands = {
-        TemplateNamespaceCommand.class,
-        TemplateValidateCommand.class,
-        TemplateExportCommand.class,
-    }
-)
-@Slf4j
-@TemplateEnabled
-public class TemplateCommand extends AbstractCommand {
-    @SneakyThrows
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        PicocliRunner.call(App.class, "template",  "--help");
-
-        return 0;
-    }
-}

cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java

@@ -1,61 +0,0 @@
-package io.kestra.cli.commands.templates;
-
-import io.kestra.cli.AbstractApiCommand;
-import io.kestra.cli.AbstractValidateCommand;
-import io.kestra.cli.services.TenantIdSelectorService;
-import io.kestra.core.models.templates.TemplateEnabled;
-import io.micronaut.http.HttpRequest;
-import io.micronaut.http.HttpResponse;
-import io.micronaut.http.MediaType;
-import io.micronaut.http.MutableHttpRequest;
-import io.micronaut.http.client.exceptions.HttpClientResponseException;
-import io.micronaut.http.client.netty.DefaultHttpClient;
-import jakarta.inject.Inject;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-@CommandLine.Command(
-    name = "export",
-    description = "Export templates to a ZIP file",
-    mixinStandardHelpOptions = true
-)
-@Slf4j
-@TemplateEnabled
-public class TemplateExportCommand extends AbstractApiCommand {
-    private static final String DEFAULT_FILE_NAME = "templates.zip";
-
-    @Inject
-    private TenantIdSelectorService tenantService;
-
-    @CommandLine.Option(names = {"--namespace"}, description = "The namespace of templates to export")
-    public String namespace;
-
-    @CommandLine.Parameters(index = "0", description = "The directory to export the file to")
-    public Path directory;
-
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        try(DefaultHttpClient client = client()) {
-            MutableHttpRequest<Object> request = HttpRequest
-                .GET(apiUri("/templates/export/by-query", tenantService.getTenantId(tenantId)) + (namespace != null ? "?namespace=" + namespace : ""))
-                .accept(MediaType.APPLICATION_OCTET_STREAM);
-
-            HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
-            Path zipFile = Path.of(directory.toString(), DEFAULT_FILE_NAME);
-            zipFile.toFile().createNewFile();
-            Files.write(zipFile, response.body());
-
-            stdOut("Exporting template(s) for namespace '" + namespace + "' successfully done !");
-        } catch (HttpClientResponseException e) {
-            AbstractValidateCommand.handleHttpException(e, "template");
-            return 1;
-        }
-
-        return 0;
-    }
-}

cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java

@@ -1,35 +0,0 @@
-package io.kestra.cli.commands.templates;
-
-import io.kestra.cli.AbstractValidateCommand;
-import io.kestra.core.models.templates.Template;
-import io.kestra.core.models.templates.TemplateEnabled;
-import io.kestra.core.models.validations.ModelValidator;
-import jakarta.inject.Inject;
-import picocli.CommandLine;
-
-import java.util.Collections;
-
-@CommandLine.Command(
-    name = "validate",
-    description = "Validate a template"
-)
-@TemplateEnabled
-public class TemplateValidateCommand extends AbstractValidateCommand {
-
-    @Inject
-    private ModelValidator modelValidator;
-
-    @Override
-    public Integer call() throws Exception {
-        return this.call(
-            Template.class,
-            modelValidator,
-            (Object object) -> {
-                Template template = (Template) object;
-                return template.getNamespace() + " / " + template.getId();
-            },
-            (Object object) -> Collections.emptyList(),
-            (Object object) -> Collections.emptyList()
-        );
-    }
-}

cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java

@@ -1,31 +0,0 @@
-package io.kestra.cli.commands.templates.namespaces;
-
-import io.kestra.cli.AbstractCommand;
-import io.kestra.cli.App;
-import io.kestra.core.models.templates.TemplateEnabled;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-@CommandLine.Command(
-    name = "namespace",
-    description = "Manage namespace templates",
-    mixinStandardHelpOptions = true,
-    subcommands = {
-        TemplateNamespaceUpdateCommand.class,
-    }
-)
-@Slf4j
-@TemplateEnabled
-public class TemplateNamespaceCommand extends AbstractCommand {
-    @SneakyThrows
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        PicocliRunner.call(App.class, "template", "namespace", "--help");
-
-        return 0;
-    }
-}

cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java

@@ -1,74 +0,0 @@
-package io.kestra.cli.commands.templates.namespaces;
-
-import io.kestra.cli.AbstractValidateCommand;
-import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
-import io.kestra.cli.services.TenantIdSelectorService;
-import io.kestra.core.models.templates.Template;
-import io.kestra.core.models.templates.TemplateEnabled;
-import io.kestra.core.serializers.YamlParser;
-import io.micronaut.core.type.Argument;
-import io.micronaut.http.HttpRequest;
-import io.micronaut.http.MutableHttpRequest;
-import io.micronaut.http.client.exceptions.HttpClientResponseException;
-import io.micronaut.http.client.netty.DefaultHttpClient;
-import jakarta.inject.Inject;
-import lombok.extern.slf4j.Slf4j;
-import picocli.CommandLine;
-
-import java.nio.file.Files;
-import java.util.List;
-
-import jakarta.validation.ConstraintViolationException;
-
-@CommandLine.Command(
-    name = "update",
-    description = "Update namespace templates",
-    mixinStandardHelpOptions = true
-)
-@Slf4j
-@TemplateEnabled
-public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
-
-    @Inject
-    private TenantIdSelectorService tenantService;
-
-    @Override
-    public Integer call() throws Exception {
-        super.call();
-
-        try (var files = Files.walk(directory)) {
-            List<Template> templates = files
-                .filter(Files::isRegularFile)
-                .filter(YamlParser::isValidExtension)
-                .map(path -> YamlParser.parse(path.toFile(), Template.class))
-                .toList();
-
-            if (templates.isEmpty()) {
-                stdOut("No template found on '{}'", directory.toFile().getAbsolutePath());
-            }
-
-            try (DefaultHttpClient client = client()) {
-                MutableHttpRequest<List<Template>> request = HttpRequest
-                    .POST(apiUri("/templates/", tenantService.getTenantIdAndAllowEETenants(tenantId)) + namespace + "?delete=" + delete, templates);
-
-                List<UpdateResult> updated = client.toBlocking().retrieve(
-                    this.requestOptions(request),
-                    Argument.listOf(UpdateResult.class)
-                );
-
-                stdOut(updated.size() + " template(s) for namespace '" + namespace + "' successfully updated !");
-                updated.forEach(template -> stdOut("- " + template.getNamespace() + "." + template.getId()));
-            } catch (HttpClientResponseException e) {
-                AbstractValidateCommand.handleHttpException(e, "template");
-
-                return 1;
-            }
-        } catch (ConstraintViolationException e) {
-            AbstractValidateCommand.handleException(e, "template");
-
-            return 1;
-        }
-
-        return 0;
-    }
-}

cli/src/main/resources/application.yml

@@ -30,15 +30,15 @@ micronaut:
     read-idle-timeout: 60m
     write-idle-timeout: 60m
     idle-timeout: 60m
+    netty:
+      max-zstd-encode-size: 67108864 # increased to 64MB from the default of 32MB
+      max-chunk-size: 10MB
+      max-header-size: 32768 # increased from the default of 8k
     responses:
       file:
         cache-seconds: 86400
         cache-control:
           public: true
-    netty:
-      max-zstd-encode-size: 67108864 # increased to 64MB from the default of 32MB
-      max-chunk-size: 10MB
-      max-header-size: 32768 # increased from the default of 8k
 
       # Access log configuration, see https://docs.micronaut.io/latest/guide/index.html#accessLogger
       access-logger:

cli/src/test/java/io/kestra/cli/commands/configs/sys/NoConfigCommandTest.java

@@ -68,8 +68,7 @@ class NoConfigCommandTest {
 
 
             assertThat(exitCode).isNotZero();
-            // check that the only log is an access log: this has the advantage to also check that access log is working!
-            assertThat(out.toString()).contains("POST /api/v1/main/flows HTTP/1.1 | status: 500");
+            assertThat(out.toString()).isEmpty();
             assertThat(err.toString()).contains("No bean of type [io.kestra.core.repositories.FlowRepositoryInterface] exists");
         }
     }

cli/src/test/java/io/kestra/cli/commands/flows/FlowDotCommandTest.java

@@ -14,7 +14,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 class FlowDotCommandTest {
     @Test
     void run()  {
-        URL directory = TemplateValidateCommandTest.class.getClassLoader().getResource("flows/same/first.yaml");
+        URL directory = FlowDotCommandTest.class.getClassLoader().getResource("flows/same/first.yaml");
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setOut(new PrintStream(out));
 

cli/src/test/java/io/kestra/cli/commands/flows/FlowExpandCommandTest.java

@@ -1,41 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class FlowExpandCommandTest {
-    @SuppressWarnings("deprecation")
-    @Test
-    void run() {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
-            String[] args = {
-                "src/test/resources/helper/include.yaml"
-            };
-            Integer call = PicocliRunner.call(FlowExpandCommand.class, ctx, args);
-
-            assertThat(call).isZero();
-            assertThat(out.toString()).isEqualTo("id: include\n" +
-                "namespace: io.kestra.cli\n" +
-                "\n" +
-                "# The list of tasks\n" +
-                "tasks:\n" +
-                "- id: t1\n" +
-                "  type: io.kestra.plugin.core.debug.Return\n" +
-                "  format: \"Lorem ipsum dolor sit amet\"\n" +
-                "- id: t2\n" +
-                "  type: io.kestra.plugin.core.debug.Return\n" +
-                "  format: |\n" +
-                "    Lorem ipsum dolor sit amet\n" +
-                "    Lorem ipsum dolor sit amet\n");
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/flows/FlowValidateCommandTest.java

@@ -61,7 +61,6 @@ class FlowValidateCommandTest {
 
             assertThat(call).isZero();
             assertThat(out.toString()).contains("✓ - system / warning");
-            assertThat(out.toString()).contains("⚠ - tasks[0] is deprecated");
             assertThat(out.toString()).contains("ℹ - io.kestra.core.tasks.log.Log is replaced by io.kestra.plugin.core.log.Log");
         }
     }

cli/src/test/java/io/kestra/cli/commands/flows/FlowsSyncFromSourceCommandTest.java

@@ -1,73 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
-import static org.assertj.core.api.Assertions.assertThat;
-
-import io.kestra.core.models.flows.Flow;
-import io.kestra.core.repositories.FlowRepositoryInterface;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import io.micronaut.context.env.Environment;
-import io.micronaut.runtime.server.EmbeddedServer;
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.net.URL;
-import java.util.List;
-import org.junit.jupiter.api.Test;
-
-class FlowsSyncFromSourceCommandTest {
-    @Test
-    void updateAllFlowsFromSource()  {
-        URL directory = FlowUpdatesCommandTest.class.getClassLoader().getResource("flows");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--plugins",
-                "/tmp", // pass this arg because it can cause failure
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "--delete",
-                directory.getPath(),
-            };
-            PicocliRunner.call(FlowUpdatesCommand.class, ctx, args);
-
-            assertThat(out.toString()).contains("successfully updated !");
-            out.reset();
-
-            FlowRepositoryInterface repository = ctx.getBean(FlowRepositoryInterface.class);
-            List<Flow> flows = repository.findAll(MAIN_TENANT);
-            for (Flow flow : flows) {
-                assertThat(flow.getRevision()).isEqualTo(1);
-            }
-
-            args = new String[]{
-                "--plugins",
-                "/tmp", // pass this arg because it can cause failure
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word"
-
-            };
-            PicocliRunner.call(FlowsSyncFromSourceCommand.class, ctx, args);
-
-            assertThat(out.toString()).contains("4 flow(s) successfully updated!");
-            assertThat(out.toString()).contains("- io.kestra.outsider.quattro");
-            assertThat(out.toString()).contains("- io.kestra.cli.second");
-            assertThat(out.toString()).contains("- io.kestra.cli.third");
-            assertThat(out.toString()).contains("- io.kestra.cli.first");
-
-            flows = repository.findAll(MAIN_TENANT);
-            for (Flow flow : flows) {
-                assertThat(flow.getRevision()).isEqualTo(2);
-            }
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/flows/TemplateValidateCommandTest.java

@@ -1,62 +0,0 @@
-package io.kestra.cli.commands.flows;
-
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import io.micronaut.context.env.Environment;
-import io.micronaut.runtime.server.EmbeddedServer;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.net.URL;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class TemplateValidateCommandTest {
-    @Test
-    void runLocal()  {
-        URL directory = TemplateValidateCommandTest.class.getClassLoader().getResource("invalids/empty.yaml");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setErr(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
-            String[] args = {
-                "--local",
-                directory.getPath()
-            };
-            Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args);
-
-            assertThat(call).isEqualTo(1);
-            assertThat(out.toString()).contains("Unable to parse flow");
-            assertThat(out.toString()).contains("must not be empty");
-        }
-    }
-
-    @Test
-    void runServer()  {
-        URL directory = TemplateValidateCommandTest.class.getClassLoader().getResource("invalids/empty.yaml");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setErr(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--plugins",
-                "/tmp", // pass this arg because it can cause failure
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                directory.getPath()
-            };
-            Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args);
-
-            assertThat(call).isEqualTo(1);
-            assertThat(out.toString()).contains("Unable to parse flow");
-            assertThat(out.toString()).contains("must not be empty");
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/migrations/metadata/MetadataMigrationServiceTest.java

@@ -1,57 +0,0 @@
-package io.kestra.cli.commands.migrations.metadata;
-
-import io.kestra.core.repositories.FlowRepositoryInterface;
-import io.kestra.core.tenant.TenantService;
-import io.kestra.core.utils.NamespaceUtils;
-import io.kestra.core.utils.TestsUtils;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class MetadataMigrationServiceTest<T extends MetadataMigrationService> {
-    private static final String TENANT_ID = TestsUtils.randomTenant();
-
-    protected static final String SYSTEM_NAMESPACE = "my.system.namespace";
-
-    @Test
-    void namespacesPerTenant() {
-        Map<String, List<String>> expected = getNamespacesPerTenant();
-        Map<String, List<String>> result = metadataMigrationService(
-            expected
-        ).namespacesPerTenant();
-
-        assertThat(result).hasSize(expected.size());
-        expected.forEach((tenantId, namespaces) -> {
-            assertThat(result.get(tenantId)).containsExactlyInAnyOrderElementsOf(
-                Stream.concat(
-                    Stream.of(SYSTEM_NAMESPACE),
-                    namespaces.stream()
-                ).map(NamespaceUtils::asTree).flatMap(Collection::stream).distinct().toList()
-            );
-        });
-    }
-
-    protected Map<String, List<String>> getNamespacesPerTenant() {
-        return Map.of(TENANT_ID, List.of("my.first.namespace", "my.second.namespace", "another.namespace"));
-    }
-
-    protected T metadataMigrationService(Map<String, List<String>> namespacesPerTenant) {
-        FlowRepositoryInterface mockedFlowRepository = Mockito.mock(FlowRepositoryInterface.class);
-        Mockito.doAnswer((params) -> namespacesPerTenant.get(params.getArgument(0).toString())).when(mockedFlowRepository).findDistinctNamespace(Mockito.anyString());
-        NamespaceUtils namespaceUtils = Mockito.mock(NamespaceUtils.class);
-        Mockito.when(namespaceUtils.getSystemFlowNamespace()).thenReturn(SYSTEM_NAMESPACE);
-        //noinspection unchecked
-        return ((T) new MetadataMigrationService(mockedFlowRepository, new TenantService() {
-            @Override
-            public String resolveTenant() {
-                return TENANT_ID;
-            }
-        }, null, null, namespaceUtils));
-    }
-}

cli/src/test/java/io/kestra/cli/commands/sys/statestore/StateStoreCommandTest.java

@@ -1,27 +0,0 @@
-package io.kestra.cli.commands.sys.statestore;
-
-import io.kestra.cli.commands.sys.database.DatabaseCommand;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class StateStoreCommandTest {
-    @Test
-    void runWithNoParam() {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
-            String[] args = {};
-            Integer call = PicocliRunner.call(StateStoreCommand.class, ctx, args);
-
-            assertThat(call).isZero();
-            assertThat(out.toString()).contains("Usage: kestra sys state-store");
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/sys/statestore/StateStoreMigrateCommandTest.java

@@ -1,75 +0,0 @@
-package io.kestra.cli.commands.sys.statestore;
-
-import io.kestra.core.exceptions.MigrationRequiredException;
-import io.kestra.core.exceptions.ResourceExpiredException;
-import io.kestra.core.models.flows.Flow;
-import io.kestra.core.models.flows.GenericFlow;
-import io.kestra.core.repositories.FlowRepositoryInterface;
-import io.kestra.core.runners.RunContext;
-import io.kestra.core.runners.RunContextFactory;
-import io.kestra.core.storages.StateStore;
-import io.kestra.core.storages.StorageInterface;
-import io.kestra.core.utils.Hashing;
-import io.kestra.core.utils.Slugify;
-import io.kestra.plugin.core.log.Log;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class StateStoreMigrateCommandTest {
-    @Test
-    void runMigration() throws IOException, ResourceExpiredException {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).environments("test").start()) {
-            FlowRepositoryInterface flowRepository = ctx.getBean(FlowRepositoryInterface.class);
-
-            Flow flow = Flow.builder()
-                .tenantId("my-tenant")
-                .id("a-flow")
-                .namespace("some.valid.namespace." + ((int) (Math.random() * 1000000)))
-                .tasks(List.of(Log.builder().id("log").type(Log.class.getName()).message("logging").build()))
-                .build();
-            flowRepository.create(GenericFlow.of(flow));
-
-            StorageInterface storage = ctx.getBean(StorageInterface.class);
-            String tenantId = flow.getTenantId();
-            URI oldStateStoreUri = URI.create("/" + flow.getNamespace().replace(".", "/") + "/" + Slugify.of("a-flow") + "/states/my-state/" + Hashing.hashToString("my-taskrun-value") + "/sub-name");
-            storage.put(
-                tenantId,
-                flow.getNamespace(),
-                oldStateStoreUri,
-                new ByteArrayInputStream("my-value".getBytes())
-            );
-            assertThat(storage.exists(tenantId, flow.getNamespace(), oldStateStoreUri)).isTrue();
-
-            RunContext runContext = ctx.getBean(RunContextFactory.class).of(flow, Map.of("flow", Map.of(
-                "tenantId", tenantId,
-                "id", flow.getId(),
-                "namespace", flow.getNamespace()
-            )));
-            StateStore stateStore = new StateStore(runContext, true);
-            Assertions.assertThrows(MigrationRequiredException.class, () -> stateStore.getState(true, "my-state", "sub-name", "my-taskrun-value"));
-
-            String[] args = {};
-            Integer call = PicocliRunner.call(StateStoreMigrateCommand.class, ctx, args);
-
-            assertThat(new String(stateStore.getState(true, "my-state", "sub-name", "my-taskrun-value").readAllBytes())).isEqualTo("my-value");
-            assertThat(storage.exists(tenantId, flow.getNamespace(), oldStateStoreUri)).isFalse();
-
-            assertThat(call).isZero();
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java

@@ -1,65 +0,0 @@
-package io.kestra.cli.commands.templates;
-
-import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceUpdateCommand;
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import io.micronaut.context.env.Environment;
-import io.micronaut.runtime.server.EmbeddedServer;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URL;
-import java.util.Map;
-import java.util.zip.ZipFile;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class TemplateExportCommandTest {
-    @Test
-    void run() throws IOException {
-        URL directory = TemplateExportCommandTest.class.getClassLoader().getResource("templates");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            // we use the update command to add templates to extract
-            String[] args = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "io.kestra.tests",
-                directory.getPath(),
-
-            };
-            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
-            assertThat(out.toString()).contains("3 template(s)");
-
-            // then we export them
-            String[] exportArgs = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "--namespace",
-                "io.kestra.tests",
-                "/tmp",
-            };
-            PicocliRunner.call(TemplateExportCommand.class, ctx, exportArgs);
-            File file = new File("/tmp/templates.zip");
-            assertThat(file.exists()).isTrue();
-            ZipFile zipFile = new ZipFile(file);
-            assertThat(zipFile.stream().count()).isEqualTo(3L);
-
-            file.delete();
-        }
-    }
-
-}

cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java

@@ -1,61 +0,0 @@
-package io.kestra.cli.commands.templates;
-
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import io.micronaut.context.env.Environment;
-import io.micronaut.runtime.server.EmbeddedServer;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.net.URL;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class TemplateValidateCommandTest {
-    @Test
-    void runLocal()  {
-        URL directory = TemplateValidateCommandTest.class.getClassLoader().getResource("invalidsTemplates/template.yml");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setErr(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-            String[] args = {
-                "--local",
-                directory.getPath()
-            };
-            Integer call = PicocliRunner.call(TemplateValidateCommand.class, ctx, args);
-
-            assertThat(call).isEqualTo(1);
-            assertThat(out.toString()).contains("Unable to parse template");
-            assertThat(out.toString()).contains("must not be empty");
-        }
-    }
-
-    @Test
-    void runServer()  {
-        URL directory = TemplateValidateCommandTest.class.getClassLoader().getResource("invalidsTemplates/template.yml");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setErr(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                directory.getPath()
-            };
-            Integer call = PicocliRunner.call(TemplateValidateCommand.class, ctx, args);
-
-            assertThat(call).isEqualTo(1);
-            assertThat(out.toString()).contains("Unable to parse template");
-            assertThat(out.toString()).contains("must not be empty");
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommandTest.java

@@ -1,26 +0,0 @@
-package io.kestra.cli.commands.templates.namespaces;
-
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class TemplateNamespaceCommandTest {
-    @Test
-    void runWithNoParam() {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
-            String[] args = {};
-            Integer call = PicocliRunner.call(TemplateNamespaceCommand.class, ctx, args);
-
-            assertThat(call).isZero();
-            assertThat(out.toString()).contains("Usage: kestra template namespace");
-        }
-    }
-}

cli/src/test/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java

@@ -1,112 +0,0 @@
-package io.kestra.cli.commands.templates.namespaces;
-
-import io.micronaut.configuration.picocli.PicocliRunner;
-import io.micronaut.context.ApplicationContext;
-import io.micronaut.context.env.Environment;
-import io.micronaut.runtime.server.EmbeddedServer;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.net.URL;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class TemplateNamespaceUpdateCommandTest {
-    @Test
-    void run() {
-        URL directory = TemplateNamespaceUpdateCommandTest.class.getClassLoader().getResource("templates");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "io.kestra.tests",
-                directory.getPath(),
-
-            };
-            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
-
-            assertThat(out.toString()).contains("3 template(s)");
-        }
-    }
-
-    @Test
-    void invalid() {
-        URL directory = TemplateNamespaceUpdateCommandTest.class.getClassLoader().getResource("invalidsTemplates");
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setErr(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "io.kestra.tests",
-                directory.getPath(),
-
-            };
-            Integer call = PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
-
-//            assertThat(call, is(1));
-            assertThat(out.toString()).contains("Unable to parse templates");
-            assertThat(out.toString()).contains("must not be empty");
-        }
-    }
-
-    @Test
-    void runNoDelete() {
-        URL directory = TemplateNamespaceUpdateCommandTest.class.getClassLoader().getResource("templates");
-        URL subDirectory = TemplateNamespaceUpdateCommandTest.class.getClassLoader().getResource("templates/templatesSubFolder");
-
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        System.setOut(new PrintStream(out));
-
-        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
-
-            EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
-            embeddedServer.start();
-
-            String[] args = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "io.kestra.tests",
-                directory.getPath(),
-
-            };
-            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, args);
-
-            assertThat(out.toString()).contains("3 template(s)");
-
-            String[] newArgs = {
-                "--server",
-                embeddedServer.getURL().toString(),
-                "--user",
-                "myuser:pass:word",
-                "io.kestra.tests",
-                subDirectory.getPath(),
-                "--no-delete"
-
-            };
-            PicocliRunner.call(TemplateNamespaceUpdateCommand.class, ctx, newArgs);
-
-            assertThat(out.toString()).contains("1 template(s)");
-        }
-    }
-}

cli/src/test/resources/warning/flow-with-warning.yaml

@@ -3,8 +3,8 @@ namespace: system
 
 tasks:
   - id: deprecated
-    type: io.kestra.plugin.core.debug.Echo
-    format: Hello World
+    type: io.kestra.plugin.core.log.Log
+    message: Hello World
   - id: alias
     type: io.kestra.core.tasks.log.Log
     message: I'm an alias

core/src/main/java/io/kestra/core/docs/JsonSchemaCache.java

@@ -4,7 +4,6 @@ import io.kestra.core.models.dashboards.Dashboard;
 import io.kestra.core.models.flows.Flow;
 import io.kestra.core.models.flows.PluginDefault;
 import io.kestra.core.models.tasks.Task;
-import io.kestra.core.models.templates.Template;
 import io.kestra.core.models.triggers.AbstractTrigger;
 import jakarta.inject.Singleton;
 
@@ -36,7 +35,6 @@ public class JsonSchemaCache {
     public JsonSchemaCache(final JsonSchemaGenerator jsonSchemaGenerator) {
         this.jsonSchemaGenerator = Objects.requireNonNull(jsonSchemaGenerator, "JsonSchemaGenerator cannot be null");
         registerClassForType(SchemaType.FLOW, Flow.class);
-        registerClassForType(SchemaType.TEMPLATE, Template.class);
         registerClassForType(SchemaType.TASK, Task.class);
         registerClassForType(SchemaType.TRIGGER, AbstractTrigger.class);
         registerClassForType(SchemaType.PLUGINDEFAULT, PluginDefault.class);

core/src/main/java/io/kestra/core/docs/JsonSchemaGenerator.java

@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableMap;
 import io.kestra.core.models.annotations.Plugin;
 import io.kestra.core.models.annotations.PluginProperty;
 import io.kestra.core.models.conditions.Condition;
-import io.kestra.core.models.conditions.ScheduleCondition;
 import io.kestra.core.models.dashboards.DataFilter;
 import io.kestra.core.models.dashboards.DataFilterKPI;
 import io.kestra.core.models.dashboards.charts.Chart;
@@ -64,7 +63,7 @@ import static io.kestra.core.serializers.JacksonMapper.MAP_TYPE_REFERENCE;
 @Singleton
 @Slf4j
 public class JsonSchemaGenerator {
-    
+
     private static final List<Class<?>> TYPES_RESOLVED_AS_STRING = List.of(Duration.class, LocalTime.class, LocalDate.class, LocalDateTime.class, ZonedDateTime.class, OffsetDateTime.class, OffsetTime.class);
     private static final List<Class<?>> SUBTYPE_RESOLUTION_EXCLUSION_FOR_PLUGIN_SCHEMA = List.of(Task.class, AbstractTrigger.class);
 
@@ -277,8 +276,8 @@ public class JsonSchemaGenerator {
             .with(Option.DEFINITION_FOR_MAIN_SCHEMA)
             .with(Option.PLAIN_DEFINITION_KEYS)
             .with(Option.ALLOF_CLEANUP_AT_THE_END);
-            
-        // HACK: Registered a custom JsonUnwrappedDefinitionProvider prior to the JacksonModule 
+
+        // HACK: Registered a custom JsonUnwrappedDefinitionProvider prior to the JacksonModule
         // to be able to return an CustomDefinition with an empty node when the ResolvedType can't be found.
         builder.forTypesInGeneral().withCustomDefinitionProvider(new JsonUnwrappedDefinitionProvider(){
             @Override
@@ -320,7 +319,7 @@ public class JsonSchemaGenerator {
         // inline some type
         builder.forTypesInGeneral()
             .withCustomDefinitionProvider(new CustomDefinitionProviderV2() {
-                
+
                 @Override
                 public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, SchemaGenerationContext context) {
                     if (javaType.isInstanceOf(Map.class) || javaType.isInstanceOf(Enum.class)) {
@@ -688,15 +687,6 @@ public class JsonSchemaGenerator {
                 .filter(Predicate.not(io.kestra.core.models.Plugin::isInternal))
                 .flatMap(clz -> safelyResolveSubtype(declaredType, clz, typeContext).stream())
                 .toList();
-        } else if (declaredType.getErasedType() == ScheduleCondition.class) {
-            return getRegisteredPlugins()
-                .stream()
-                .flatMap(registeredPlugin -> registeredPlugin.getConditions().stream())
-                .filter(ScheduleCondition.class::isAssignableFrom)
-                .filter(p -> allowedPluginTypes.isEmpty() || allowedPluginTypes.contains(p.getName()))
-                .filter(Predicate.not(io.kestra.core.models.Plugin::isInternal))
-                .flatMap(clz -> safelyResolveSubtype(declaredType, clz, typeContext).stream())
-                .toList();
         } else if (declaredType.getErasedType() == TaskRunner.class) {
             return getRegisteredPlugins()
                 .stream()

core/src/main/java/io/kestra/core/docs/Plugin.java

@@ -3,7 +3,6 @@ package io.kestra.core.docs;
 import io.kestra.core.models.annotations.PluginSubGroup;
 import io.kestra.core.plugins.RegisteredPlugin;
 import io.micronaut.core.annotation.Nullable;
-import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -118,17 +117,10 @@ public class Plugin {
             .filter(not(io.kestra.core.models.Plugin::isInternal))
             .filter(clazzFilter)
             .filter(c -> !c.getName().startsWith("org.kestra."))
-            .map(c -> {
-                Schema schema = c.getAnnotation(Schema.class);
-
-                var title = Optional.ofNullable(schema).map(Schema::title).filter(t -> !t.isEmpty()).orElse(null);
-                var description = Optional.ofNullable(schema).map(Schema::description).filter(d -> !d.isEmpty()).orElse(null);
-                var deprecated = io.kestra.core.models.Plugin.isDeprecated(c) ? true : null;
-
-                return new PluginElementMetadata(c.getName(), deprecated, title, description);
-            })
+            .map(c -> new PluginElementMetadata(c.getName(), io.kestra.core.models.Plugin.isDeprecated(c) ? true : null))
             .toList();
     }
 
-    public record PluginElementMetadata(String cls, Boolean deprecated, String title, String description) {}
+    public record PluginElementMetadata(String cls, Boolean deprecated) {
+    }
 }

core/src/main/java/io/kestra/core/docs/SchemaType.java

@@ -6,7 +6,6 @@ import io.kestra.core.utils.Enums;
 
 public enum SchemaType {
     FLOW,
-    TEMPLATE,
     TASK,
     TRIGGER,
     PLUGINDEFAULT,

core/src/main/java/io/kestra/core/exceptions/FlowNotFoundException.java

@@ -1,23 +0,0 @@
-package io.kestra.core.exceptions;
-
-/**
- * Exception that can be thrown when a Flow is not found.
- */
-public class FlowNotFoundException extends NotFoundException {
-
-    /**
-     * Creates a new {@link FlowNotFoundException} instance.
-     */
-    public FlowNotFoundException() {
-        super();
-    }
-
-    /**
-     * Creates a new {@link NotFoundException} instance.
-     *
-     * @param message the error message.
-     */
-    public FlowNotFoundException(final String message) {
-        super(message);
-    }
-}

core/src/main/java/io/kestra/core/lock/Lock.java

@@ -0,0 +1,27 @@
+package io.kestra.core.lock;
+
+import io.kestra.core.models.HasUID;
+import io.kestra.core.utils.IdUtils;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+
+@Getter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Lock implements HasUID {
+    private String category;
+    private String id;
+    private String owner;
+    private Instant createdAt;
+
+    @Override
+    public String uid() {
+        return IdUtils.fromParts(this.category, this.id);
+    }
+}

core/src/main/java/io/kestra/core/lock/LockException.java

@@ -0,0 +1,13 @@
+package io.kestra.core.lock;
+
+import io.kestra.core.exceptions.KestraRuntimeException;
+
+public class LockException extends KestraRuntimeException {
+    public LockException(String message) {
+        super(message);
+    }
+
+    public LockException(Throwable cause) {
+        super(cause);
+    }
+}

core/src/main/java/io/kestra/core/lock/LockService.java

@@ -0,0 +1,195 @@
+package io.kestra.core.lock;
+
+import io.kestra.core.repositories.LockRepositoryInterface;
+import io.kestra.core.server.ServerInstance;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+import lombok.extern.slf4j.Slf4j;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+/**
+ * This service provides facility for executing Runnable and Callable tasks inside a lock.
+ * Note: it may be handy to provide a tryLock facility that, if locked, skips executing the Runnable or Callable and exits immediately.
+ *
+ * @implNote There is no expiry for locks, so a service may hold a lock infinitely until the service is restarted as the
+ *           liveness mechanism releases all locks when the service is unreachable.
+ *           This may be improved at some point by adding an expiry (for ex 30s) and running a thread that will periodically
+ *           increase the expiry for all exiting locks. This should allow quicker recovery of zombie locks than relying on the liveness mechanism,
+ *           as a service wanted to lock an expired lock would be able to take it over.
+ */
+@Slf4j
+@Singleton
+public class LockService {
+    private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(300);
+    private static final int DEFAULT_SLEEP_MS = 1;
+
+    private final LockRepositoryInterface lockRepository;
+
+    @Inject
+    public LockService(LockRepositoryInterface lockRepository) {
+        this.lockRepository = lockRepository;
+    }
+
+    /**
+     * Executes a Runnable inside a lock.
+     * If the lock is already taken, it will wait for at most the default lock timeout of 5mn.
+     * @see #doInLock(String, String, Duration, Runnable)
+     *
+     * @param category lock category, ex 'executions'
+     * @param id identifier of the lock identity inside the category, ex an execution ID
+     *
+     * @throws LockException if the lock cannot be hold before the timeout or the thread is interrupted.
+     */
+    public void doInLock(String category, String id, Runnable runnable) {
+        doInLock(category, id, DEFAULT_TIMEOUT, runnable);
+    }
+
+    /**
+     * Executes a Runnable inside a lock.
+     * If the lock is already taken, it will wait for at most the <code>timeout</code> duration.
+     * @see #doInLock(String, String, Runnable)
+     *
+     * @param category lock category, ex 'executions'
+     * @param id identifier of the lock identity inside the category, ex an execution ID
+     * @param timeout how much time to wait for the lock if another process already holds the same lock
+     *
+     * @throws LockException if the lock cannot be hold before the timeout or the thread is interrupted.
+     */
+    public void doInLock(String category, String id, Duration timeout, Runnable runnable) {
+        if (!lock(category, id, timeout)) {
+            throw new LockException("Unable to hold the lock inside the configured timeout of " + timeout);
+        }
+
+        try {
+            runnable.run();
+        } finally {
+            unlock(category, id);
+        }
+    }
+
+    /**
+     * Attempts to execute the provided {@code runnable} within a lock.
+     * If the lock is already held by another process, the execution is skipped.
+     *
+     * @param category the category of the lock, e.g., 'executions'
+     * @param id the identifier of the lock within the specified category, e.g., an execution ID
+     * @param runnable the task to be executed if the lock is successfully acquired
+     */
+    public void tryLock(String category, String id, Runnable runnable) {
+        if (lock(category, id, Duration.ZERO)) {
+            try {
+                runnable.run();
+            } finally {
+                unlock(category, id);
+            }
+        } else {
+            log.debug("Lock '{}'.'{}' already hold, skipping", category, id);
+        }
+    }
+
+    /**
+     * Executes a Callable inside a lock.
+     * If the lock is already taken, it will wait for at most the default lock timeout of 5mn.
+     *
+     * @param category lock category, ex 'executions'
+     * @param id identifier of the lock identity inside the category, ex an execution ID
+     *
+     * @throws LockException if the lock cannot be hold before the timeout or the thread is interrupted.
+     */
+    public <T> T callInLock(String category, String id, Callable<T> callable) throws Exception {
+        return callInLock(category, id, DEFAULT_TIMEOUT, callable);
+    }
+
+    /**
+     * Executes a Callable inside a lock.
+     * If the lock is already taken, it will wait for at most the <code>timeout</code> duration.
+     *
+     * @param category lock category, ex 'executions'
+     * @param id identifier of the lock identity inside the category, ex an execution ID
+     * @param timeout how much time to wait for the lock if another process already holds the same lock
+     *
+     * @throws LockException if the lock cannot be hold before the timeout or the thread is interrupted.
+     */
+    public <T> T callInLock(String category, String id, Duration timeout, Callable<T> callable) throws Exception {
+        if (!lock(category, id, timeout)) {
+            throw new LockException("Unable to hold the lock inside the configured timeout of " + timeout);
+        }
+
+        try {
+            return callable.call();
+        } finally {
+            unlock(category, id);
+        }
+    }
+
+    /**
+     * Release all locks hold by this service identifier.
+     */
+    public List<Lock> releaseAllLocks(String serviceId) {
+        return lockRepository.deleteByOwner(serviceId);
+    }
+
+    /**
+     * @return true if the lock identified by this category and identifier already exist.
+     */
+    public boolean isLocked(String category, String id) {
+        return lockRepository.findById(category, id).isPresent();
+    }
+
+    private boolean lock(String category, String id, Duration timeout) throws LockException {
+        log.debug("Locking '{}'.'{}'", category,  id);
+        long deadline = System.currentTimeMillis() + timeout.toMillis();
+        do {
+            Optional<Lock> existing = lockRepository.findById(category, id);
+            if (existing.isEmpty()) {
+                // we can try to lock!
+                Lock newLock = new Lock(category, id, ServerInstance.INSTANCE_ID, Instant.now());
+                if (lockRepository.create(newLock)) {
+                    return true;
+                } else {
+                    log.debug("Cannot create the lock, it may have been created after we check for its existence and before we create it");
+                }
+            } else {
+                log.debug("Already locked by: {}", existing.get().getOwner());
+            }
+
+            // fast path for when we don't want to wait for the lock
+            if (timeout.isZero()) {
+                return false;
+            }
+
+            try {
+                Thread.sleep(DEFAULT_SLEEP_MS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new LockException(e);
+            }
+        } while (System.currentTimeMillis() < deadline);
+
+        log.debug("Lock already hold, waiting for it to be released");
+        return false;
+    }
+
+    private void unlock(String category, String id) {
+        log.debug("Unlocking '{}'.'{}'", category, id);
+
+        Optional<Lock> existing = lockRepository.findById(category, id);
+        if (existing.isEmpty()) {
+            log.warn("Try to unlock unknown lock '{}'.'{}', ignoring it", category, id);
+            return;
+        }
+
+        if (!existing.get().getOwner().equals(ServerInstance.INSTANCE_ID)) {
+            log.warn("Try to unlock a lock we no longer own '{}'.'{}', ignoring it", category, id);
+            return;
+        }
+
+        lockRepository.deleteById(category, id);
+    }
+}

core/src/main/java/io/kestra/core/models/HasSource.java

@@ -7,6 +7,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -64,7 +65,7 @@ public interface HasSource {
 
             if (isYAML(fileName)) {
                 byte[] bytes = inputStream.readAllBytes();
-                List<String> sources = List.of(new String(bytes).split("(?m)^---\\s*$"));
+                List<String> sources = List.of(new String(bytes).split("---"));
                 for (int i = 0; i < sources.size(); i++) {
                     String source = sources.get(i);
                     reader.accept(source, String.valueOf(i));

core/src/main/java/io/kestra/core/models/conditions/ScheduleCondition.java

@@ -2,7 +2,11 @@ package io.kestra.core.models.conditions;
 
 import io.kestra.core.exceptions.InternalException;
 
-
+/**
+ * Conditions of type ScheduleCondition have a special behavior inside the {@link io.kestra.plugin.core.trigger.Schedule} trigger.
+ * They are evaluated specifically and would be taken into account when computing the next evaluation date.
+ * Only conditions based on date should be marked as ScheduleCondition.
+ */
 public interface ScheduleCondition {
     boolean test(ConditionContext conditionContext) throws InternalException;
 }

core/src/main/java/io/kestra/core/models/dashboards/Dashboard.java

@@ -6,6 +6,7 @@ import io.kestra.core.models.HasUID;
 import io.kestra.core.models.dashboards.charts.Chart;
 import io.kestra.core.utils.IdUtils;
 import io.micronaut.core.annotation.Introspected;
+import io.swagger.v3.oas.annotations.Hidden;
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
@@ -26,9 +27,11 @@ import java.util.Objects;
 @Introspected
 @ToString
 public class Dashboard implements HasUID, DeletedInterface {
+    @Hidden
     @Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
     private String tenantId;
 
+    @Hidden
     @NotNull
     @NotBlank
     private String id;
@@ -46,12 +49,15 @@ public class Dashboard implements HasUID, DeletedInterface {
     @Valid
     private List<Chart<?>> charts;
 
+    @Hidden
     @NotNull
     @Builder.Default
     private boolean deleted = false;
 
+    @Hidden
     private Instant created;
 
+    @Hidden
     private Instant updated;
 
     private String sourceCode;

core/src/main/java/io/kestra/core/models/dashboards/DataFilter.java

@@ -5,8 +5,6 @@ import io.kestra.core.models.annotations.Plugin;
 import io.kestra.core.models.dashboards.filters.AbstractFilter;
 import io.kestra.core.repositories.QueryBuilderInterface;
 import io.kestra.plugin.core.dashboard.data.IData;
-import jakarta.annotation.Nullable;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
 import jakarta.validation.constraints.Pattern;
@@ -35,12 +33,9 @@ public abstract class DataFilter<F extends Enum<F>, C extends ColumnDescriptor<F
     @Pattern(regexp = JAVA_IDENTIFIER_REGEX)
     private String type;
 
-    @Valid
     private Map<String, C> columns;
 
     @Setter
-    @Valid
-    @Nullable
     private List<AbstractFilter<F>> where;
 
     private List<OrderBy> orderBy;

core/src/main/java/io/kestra/core/models/dashboards/charts/DataChart.java

@@ -5,7 +5,6 @@ import io.kestra.core.models.annotations.Plugin;
 import io.kestra.core.models.dashboards.ChartOption;
 import io.kestra.core.models.dashboards.DataFilter;
 import io.kestra.core.validations.DataChartValidation;
-import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotNull;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -21,7 +20,6 @@ import lombok.experimental.SuperBuilder;
 @DataChartValidation
 public abstract class DataChart<P extends ChartOption, D extends DataFilter<?, ?>> extends Chart<P> implements io.kestra.core.models.Plugin {
     @NotNull
-    @Valid
     private D data;
 
     public Integer minNumberOfAggregations() {

core/src/main/java/io/kestra/core/models/dashboards/filters/AbstractFilter.java

@@ -1,11 +1,8 @@
 package io.kestra.core.models.dashboards.filters;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.micronaut.core.annotation.Introspected;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.NotNull;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.SuperBuilder;
@@ -35,9 +32,6 @@ import lombok.experimental.SuperBuilder;
 @SuperBuilder
 @Introspected
 public abstract class AbstractFilter<F extends Enum<F>> {
-    @NotNull
-    @JsonProperty(value = "field", required = true)
-    @Valid
     private F field;
     private String labelKey;
 

core/src/main/java/io/kestra/core/models/executions/Execution.java

@@ -658,20 +658,18 @@ public class Execution implements DeletedInterface, TenantInterface {
     public boolean hasFailedNoRetry(List<ResolvedTask> resolvedTasks, TaskRun parentTaskRun) {
         return this.findTaskRunByTasks(resolvedTasks, parentTaskRun)
             .stream()
-            // NOTE: we check on isFailed first to avoid the costly shouldBeRetried() method
-            .anyMatch(taskRun -> taskRun.getState().isFailed() && shouldNotBeRetried(resolvedTasks, parentTaskRun, taskRun));
-    }
-
-    private static boolean shouldNotBeRetried(List<ResolvedTask> resolvedTasks, TaskRun parentTaskRun, TaskRun taskRun) {
-        ResolvedTask resolvedTask = resolvedTasks.stream()
-            .filter(t -> t.getTask().getId().equals(taskRun.getTaskId())).findFirst()
-            .orElse(null);
-        if (resolvedTask == null) {
-            log.warn("Can't find task for taskRun '{}' in parentTaskRun '{}'",
-                taskRun.getId(), parentTaskRun.getId());
-            return false;
-        }
-        return !taskRun.shouldBeRetried(resolvedTask.getTask().getRetry());
+            .anyMatch(taskRun -> {
+                ResolvedTask resolvedTask = resolvedTasks.stream()
+                    .filter(t -> t.getTask().getId().equals(taskRun.getTaskId())).findFirst()
+                    .orElse(null);
+                if (resolvedTask == null) {
+                    log.warn("Can't find task for taskRun '{}' in parentTaskRun '{}'",
+                        taskRun.getId(), parentTaskRun.getId());
+                    return false;
+                }
+                return !taskRun.shouldBeRetried(resolvedTask.getTask().getRetry())
+                    && taskRun.getState().isFailed();
+            });
     }
 
     public boolean hasCreated() {

core/src/main/java/io/kestra/core/models/executions/ExecutionTrigger.java

@@ -1,16 +1,15 @@
 package io.kestra.core.models.executions;
 
-import io.kestra.core.models.tasks.Output;
-import io.kestra.core.models.triggers.AbstractTrigger;
 import io.micronaut.core.annotation.Introspected;
-import io.swagger.v3.oas.annotations.media.Schema;
-import jakarta.validation.constraints.NotNull;
 import lombok.Builder;
 import lombok.Value;
+import io.kestra.core.models.tasks.Output;
+import io.kestra.core.models.triggers.AbstractTrigger;
 
 import java.net.URI;
 import java.util.Collections;
 import java.util.Map;
+import jakarta.validation.constraints.NotNull;
 
 @Value
 @Builder
@@ -22,7 +21,6 @@ public class ExecutionTrigger {
     @NotNull
     String type;
 
-    @Schema(type = "object", additionalProperties = Schema.AdditionalPropertiesValue.TRUE)
     Map<String, Object> variables;
 
     URI logFile;

core/src/main/java/io/kestra/core/models/executions/TaskRun.java

@@ -3,7 +3,9 @@ package io.kestra.core.models.executions;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import io.kestra.core.models.TenantInterface;
 import io.kestra.core.models.flows.State;
+import io.kestra.core.models.tasks.FlowableTask;
 import io.kestra.core.models.tasks.ResolvedTask;
+import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.tasks.retrys.AbstractRetry;
 import io.kestra.core.utils.IdUtils;
 import io.swagger.v3.oas.annotations.Hidden;
@@ -93,16 +95,8 @@ public class TaskRun implements TenantInterface {
             this.forceExecution
         );
     }
-    public TaskRun withStateAndAttempt(State.Type state) {
-        List<TaskRunAttempt> newAttempts = new ArrayList<>(this.attempts != null ? this.attempts : List.of());
-
-        if (newAttempts.isEmpty()) {
-            newAttempts.add(TaskRunAttempt.builder().state(new State(state)).build());
-        } else {
-            TaskRunAttempt updatedLast = newAttempts.getLast().withState(state);
-            newAttempts.set(newAttempts.size() - 1, updatedLast);
-        }
 
+    public TaskRun replaceState(State newState) {
         return new TaskRun(
             this.tenantId,
             this.id,
@@ -112,9 +106,9 @@ public class TaskRun implements TenantInterface {
             this.taskId,
             this.parentTaskRunId,
             this.value,
-            newAttempts,
+            this.attempts,
             this.outputs,
-            this.state.withState(state),
+            newState,
             this.iteration,
             this.dynamic,
             this.forceExecution
@@ -320,11 +314,4 @@ public class TaskRun implements TenantInterface {
             .build();
     }
 
-    public TaskRun addAttempt(TaskRunAttempt attempt) {
-        if (this.attempts == null) {
-            this.attempts = new ArrayList<>();
-        }
-        this.attempts.add(attempt);
-        return this;
-    }
 }

core/src/main/java/io/kestra/core/models/flows/Concurrency.java

@@ -24,8 +24,4 @@ public class Concurrency {
     public enum Behavior {
         QUEUE, CANCEL, FAIL;
     }
-
-    public static boolean possibleTransitions(State.Type type) {
-        return type.equals(State.Type.CANCELLED) || type.equals(State.Type.FAILED);
-    }
 }

core/src/main/java/io/kestra/core/models/flows/Flow.java

@@ -1,5 +1,6 @@
 package io.kestra.core.models.flows;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -11,7 +12,6 @@ import io.kestra.core.exceptions.InternalException;
 import io.kestra.core.models.HasUID;
 import io.kestra.core.models.annotations.PluginProperty;
 import io.kestra.core.models.flows.sla.SLA;
-import io.kestra.core.models.listeners.Listener;
 import io.kestra.core.models.tasks.FlowableTask;
 import io.kestra.core.models.tasks.Task;
 import io.kestra.core.models.tasks.retrys.AbstractRetry;
@@ -84,10 +84,6 @@ public class Flow extends AbstractFlow implements HasUID {
         return this._finally;
     }
 
-    @Valid
-    @Deprecated
-    List<Listener> listeners;
-
     @Valid
     List<Task> afterExecution;
 
@@ -97,20 +93,6 @@ public class Flow extends AbstractFlow implements HasUID {
     @Valid
     List<PluginDefault> pluginDefaults;
 
-    @Valid
-    List<PluginDefault> taskDefaults;
-
-    @Deprecated
-    public void setTaskDefaults(List<PluginDefault> taskDefaults) {
-        this.pluginDefaults = taskDefaults;
-        this.taskDefaults = taskDefaults;
-    }
-
-    @Deprecated
-    public List<PluginDefault> getTaskDefaults() {
-        return this.taskDefaults;
-    }
-
     @Valid
     Concurrency concurrency;
 
@@ -143,7 +125,7 @@ public class Flow extends AbstractFlow implements HasUID {
                 this.tasks != null ? this.tasks : Collections.<Task>emptyList(),
                 this.errors != null ? this.errors : Collections.<Task>emptyList(),
                 this._finally != null ? this._finally : Collections.<Task>emptyList(),
-                this.afterExecutionTasks()
+                this.afterExecution != null ? this.afterExecution : Collections.<Task>emptyList()
             )
             .flatMap(Collection::stream);
     }
@@ -244,55 +226,6 @@ public class Flow extends AbstractFlow implements HasUID {
             .orElse(null);
     }
 
-    /**
-     * @deprecated should not be used
-     */
-    @Deprecated(forRemoval = true, since = "0.21.0")
-    public Flow updateTask(String taskId, Task newValue) throws InternalException {
-        Task task = this.findTaskByTaskId(taskId);
-        Flow flow = this instanceof FlowWithSource flowWithSource ? flowWithSource.toFlow() : this;
-
-        Map<String, Object> map = NON_DEFAULT_OBJECT_MAPPER.convertValue(flow, JacksonMapper.MAP_TYPE_REFERENCE);
-
-        return NON_DEFAULT_OBJECT_MAPPER.convertValue(
-            recursiveUpdate(map, task, newValue),
-            Flow.class
-        );
-    }
-
-    private static Object recursiveUpdate(Object object, Task previous, Task newValue) {
-        if (object instanceof Map<?, ?> value) {
-            if (value.containsKey("id") && value.get("id").equals(previous.getId()) &&
-                value.containsKey("type") && value.get("type").equals(previous.getType())
-            ) {
-                return NON_DEFAULT_OBJECT_MAPPER.convertValue(newValue, JacksonMapper.MAP_TYPE_REFERENCE);
-            } else {
-                return value
-                    .entrySet()
-                    .stream()
-                    .map(e -> new AbstractMap.SimpleEntry<>(
-                        e.getKey(),
-                        recursiveUpdate(e.getValue(), previous, newValue)
-                    ))
-                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-            }
-        } else if (object instanceof Collection<?> value) {
-            return value
-                .stream()
-                .map(r -> recursiveUpdate(r, previous, newValue))
-                .toList();
-        } else {
-            return object;
-        }
-    }
-
-    private List<Task> afterExecutionTasks() {
-        return ListUtils.concat(
-            ListUtils.emptyOnNull(this.getListeners()).stream().flatMap(listener -> listener.getTasks().stream()).toList(),
-            this.getAfterExecution()
-        );
-    }
-
     public boolean equalsWithoutRevision(FlowInterface o) {
         try {
             return WITHOUT_REVISION_OBJECT_MAPPER.writeValueAsString(this).equals(WITHOUT_REVISION_OBJECT_MAPPER.writeValueAsString(o));
@@ -345,7 +278,7 @@ public class Flow extends AbstractFlow implements HasUID {
      * To be conservative a flow MUST not return any source.
      */
     @Override
-    @Schema(hidden = true)
+    @JsonIgnore
     public String getSource() {
         return null;
     }

core/src/main/java/io/kestra/core/models/flows/FlowWithSource.java

@@ -1,12 +1,14 @@
 package io.kestra.core.models.flows;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.micronaut.core.annotation.Introspected;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.ToString;
 import lombok.experimental.SuperBuilder;
-import io.swagger.v3.oas.annotations.media.Schema;
 
+import java.util.Objects;
+import java.util.regex.Pattern;
 
 @SuperBuilder(toBuilder = true)
 @Getter
@@ -17,7 +19,6 @@ public class FlowWithSource extends Flow {
 
     String source;
 
-    @SuppressWarnings("deprecation")
     public Flow toFlow() {
         return Flow.builder()
             .tenantId(this.tenantId)
@@ -32,7 +33,6 @@ public class FlowWithSource extends Flow {
             .tasks(this.tasks)
             .errors(this.errors)
             ._finally(this._finally)
-            .listeners(this.listeners)
             .afterExecution(this.afterExecution)
             .triggers(this.triggers)
             .pluginDefaults(this.pluginDefaults)
@@ -45,7 +45,7 @@ public class FlowWithSource extends Flow {
     }
 
     @Override
-    @Schema(hidden = false)
+    @JsonIgnore(value = false)
     public String getSource() {
         return this.source;
     }
@@ -58,7 +58,6 @@ public class FlowWithSource extends Flow {
             .build();
     }
 
-    @SuppressWarnings("deprecation")
     public static FlowWithSource of(Flow flow, String source) {
         return FlowWithSource.builder()
             .tenantId(flow.tenantId)
@@ -74,7 +73,6 @@ public class FlowWithSource extends Flow {
             .errors(flow.errors)
             ._finally(flow._finally)
             .afterExecution(flow.afterExecution)
-            .listeners(flow.listeners)
             .triggers(flow.triggers)
             .pluginDefaults(flow.pluginDefaults)
             .disabled(flow.disabled)

core/src/main/java/io/kestra/core/models/flows/Input.java

@@ -1,6 +1,5 @@
 package io.kestra.core.models.flows;
 
-import com.fasterxml.jackson.annotation.JsonSetter;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.kestra.core.models.flows.input.*;
@@ -26,7 +25,6 @@ import lombok.experimental.SuperBuilder;
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY)
 @JsonSubTypes({
     @JsonSubTypes.Type(value = ArrayInput.class, name = "ARRAY"),
-    @JsonSubTypes.Type(value = BooleanInput.class, name = "BOOLEAN"),
     @JsonSubTypes.Type(value = BoolInput.class, name = "BOOL"),
     @JsonSubTypes.Type(value = DateInput.class, name = "DATE"),
     @JsonSubTypes.Type(value = DateTimeInput.class, name = "DATETIME"),
@@ -37,7 +35,6 @@ import lombok.experimental.SuperBuilder;
     @JsonSubTypes.Type(value = JsonInput.class, name = "JSON"),
     @JsonSubTypes.Type(value = SecretInput.class, name = "SECRET"),
     @JsonSubTypes.Type(value = StringInput.class, name = "STRING"),
-    @JsonSubTypes.Type(value = EnumInput.class, name = "ENUM"),
     @JsonSubTypes.Type(value = SelectInput.class, name = "SELECT"),
     @JsonSubTypes.Type(value = TimeInput.class, name = "TIME"),
     @JsonSubTypes.Type(value = URIInput.class, name = "URI"),
@@ -55,9 +52,6 @@ public abstract class Input<T> implements Data {
     @Pattern(regexp="^[a-zA-Z0-9][.a-zA-Z0-9_-]*")
     String id;
 
-    @Deprecated
-    String name;
-
     @Schema(
         title = "The type of the input."
     )
@@ -95,13 +89,4 @@ public abstract class Input<T> implements Data {
     String displayName;
 
     public abstract void validate(T input) throws ConstraintViolationException;
-
-    @JsonSetter
-    public void setName(String name) {
-        if (this.id == null) {
-            this.id = name;
-        }
-
-        this.name = name;
-    }
 }

core/src/main/java/io/kestra/core/models/flows/State.java

@@ -255,10 +255,6 @@ public class State {
             return this == Type.RUNNING || this == Type.KILLING;
         }
 
-        public boolean onlyRunning() {
-            return this == Type.RUNNING;
-        }
-
         public boolean isFailed() {
             return this == Type.FAILED;
         }

core/src/main/java/io/kestra/core/models/flows/Type.java

@@ -9,11 +9,9 @@ import io.micronaut.core.annotation.Introspected;
 @Introspected
 public enum Type {
     STRING(StringInput.class.getName()),
-    ENUM(EnumInput.class.getName()),
     SELECT(SelectInput.class.getName()),
     INT(IntInput.class.getName()),
     FLOAT(FloatInput.class.getName()),
-    BOOLEAN(BooleanInput.class.getName()),
     BOOL(BoolInput.class.getName()),
     DATETIME(DateTimeInput.class.getName()),
     DATE(DateInput.class.getName()),

core/src/main/java/io/kestra/core/models/flows/input/BooleanInput.java

@@ -1,19 +0,0 @@
-package io.kestra.core.models.flows.input;
-
-import io.kestra.core.models.flows.Input;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.SuperBuilder;
-
-import jakarta.validation.ConstraintViolationException;
-
-@SuperBuilder
-@Getter
-@NoArgsConstructor
-@Deprecated
-public class BooleanInput extends Input<Boolean> {
-    @Override
-    public void validate(Boolean input) throws ConstraintViolationException {
-        // no validation yet
-    }
-}

core/src/main/java/io/kestra/core/models/flows/input/EnumInput.java

@@ -1,39 +0,0 @@
-package io.kestra.core.models.flows.input;
-
-import io.kestra.core.models.flows.Input;
-import io.kestra.core.models.validations.ManualConstraintViolation;
-import io.kestra.core.validations.Regex;
-import io.swagger.v3.oas.annotations.media.Schema;
-import jakarta.validation.ConstraintViolationException;
-import jakarta.validation.constraints.NotNull;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.SuperBuilder;
-
-import java.util.List;
-
-@SuperBuilder
-@Getter
-@NoArgsConstructor
-@Deprecated
-public class EnumInput extends Input<String> {
-    @Schema(
-        title = "List of values.",
-        description = "DEPRECATED; use 'SELECT' instead."
-    )
-    @NotNull
-    List<@Regex String> values;
-
-    @Override
-    public void validate(String input) throws ConstraintViolationException {
-        if (!values.contains(input) && this.getRequired()) {
-            throw ManualConstraintViolation.toConstraintViolationException(
-                "it must match the values `" + values + "`",
-                this,
-                EnumInput.class,
-                getId(),
-                input
-            );
-        }
-    }
-}

core/src/main/java/io/kestra/core/models/flows/input/FileInput.java

@@ -4,8 +4,6 @@ import java.util.Set;
 import io.kestra.core.models.flows.Input;
 import io.kestra.core.validations.FileInputValidation;
 import jakarta.validation.ConstraintViolationException;
-import jakarta.validation.constraints.NotNull;
-import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.experimental.SuperBuilder;
@@ -19,17 +17,14 @@ import java.util.List;
 @FileInputValidation
 public class FileInput extends Input<URI> {
 
-    private static final String DEFAULT_EXTENSION = ".upl";
+    public static final String DEFAULT_EXTENSION = ".upl";
 
-    @Deprecated(since = "0.24", forRemoval = true)
-    public String extension;
-    
     /**
      * List of allowed file extensions (e.g., [".csv", ".txt", ".pdf"]).
      * Each extension must start with a dot.
      */
     private List<String> allowedFileExtensions;
-    
+
     /**
      * Gets the file extension from the URI's path
      */
@@ -53,15 +48,4 @@ public class FileInput extends Input<URI> {
             );
         }
     }
-
-    public static String findFileInputExtension(@NotNull final List<Input<?>> inputs, @NotNull final String fileName) {
-        String res = inputs.stream()
-            .filter(in -> in instanceof FileInput)
-            .filter(in -> in.getId().equals(fileName))
-            .filter(flowInput -> ((FileInput) flowInput).getExtension() != null)
-            .map(flowInput -> ((FileInput) flowInput).getExtension())
-            .findFirst()
-            .orElse(FileInput.DEFAULT_EXTENSION);
-        return res.startsWith(".") ? res : "." + res;
-    }
 }

core/src/main/java/io/kestra/core/models/flows/sla/SLAMonitorStorage.java

@@ -1,12 +0,0 @@
-package io.kestra.core.models.flows.sla;
-
-import java.time.Instant;
-import java.util.function.Consumer;
-
-public interface SLAMonitorStorage {
-    void save(SLAMonitor slaMonitor);
-
-    void purge(String executionId);
-
-    void processExpired(Instant now, Consumer<SLAMonitor> consumer);
-}

core/src/main/java/io/kestra/core/models/listeners/Listener.java

@@ -1,25 +0,0 @@
-package io.kestra.core.models.listeners;
-
-import io.micronaut.core.annotation.Introspected;
-import lombok.Builder;
-import lombok.Value;
-import io.kestra.core.models.conditions.Condition;
-import io.kestra.core.models.tasks.Task;
-
-import java.util.List;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.NotEmpty;
-
-@Value
-@Builder
-@Introspected
-public class Listener {
-    String description;
-
-    @Valid
-    List<Condition> conditions;
-
-    @Valid
-    @NotEmpty
-    List<Task> tasks;
-}

core/src/main/java/io/kestra/core/models/property/Property.java

@@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer;
 import com.google.common.annotations.VisibleForTesting;
 import io.kestra.core.exceptions.IllegalVariableEvaluationException;
 import io.kestra.core.runners.RunContext;
-import io.kestra.core.runners.RunContextProperty;
 import io.kestra.core.serializers.JacksonMapper;
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.constraints.NotNull;
@@ -36,6 +35,7 @@ import static io.kestra.core.utils.Rethrow.throwFunction;
 @JsonDeserialize(using = Property.PropertyDeserializer.class)
 @JsonSerialize(using = Property.PropertySerializer.class)
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor(access = AccessLevel.PACKAGE)
 @Schema(
     oneOf = {
@@ -51,33 +51,17 @@ public class Property<T> {
         .copy()
         .configure(SerializationFeature.WRITE_DURATIONS_AS_TIMESTAMPS, false);
 
-    private final boolean skipCache;
     private String expression;
     private T value;
 
-    /**
-     * @deprecated use {@link #ofExpression(String)} instead.
-     */
-    @Deprecated
-    // Note: when not used, this constructor would not be deleted but made private so it can only be used by ofExpression(String) and the deserializer
-    public Property(String expression) {
-        this(expression, false);
-    }
-
-    private Property(String expression, boolean skipCache) {
+    private Property(String expression) {
         this.expression = expression;
-        this.skipCache = skipCache;
     }
 
-    /**
-     * @deprecated use {@link #ofValue(Object)} instead.
-     */
     @VisibleForTesting
-    @Deprecated
     public Property(Map<?, ?> map) {
         try {
             expression = MAPPER.writeValueAsString(map);
-            this.skipCache = false;
         } catch (JsonProcessingException e) {
             throw new IllegalArgumentException(e);
         }
@@ -90,11 +74,14 @@ public class Property<T> {
     /**
      * Returns a new {@link Property} with no cached rendered value,
      * so that the next render will evaluate its original Pebble expression.
+     * <p>
+     * The returned property will still cache its rendered result.
+     * To re-evaluate on a subsequent render, call {@code skipCache()} again.
      *
      * @return a new {@link Property} without a pre-rendered value
      */
     public Property<T> skipCache() {
-        return new Property<>(expression, true);
+        return Property.ofExpression(expression);
     }
 
     /**
@@ -131,17 +118,8 @@ public class Property<T> {
         return p;
     }
 
-    /**
-     * @deprecated use {@link #ofValue(Object)} instead.
-     */
-    @Deprecated
-    public static <V> Property<V> of(V value) {
-        return ofValue(value);
-    }
-
     /**
      * Build a new Property object with a Pebble expression.<br>
-     * This property object will not cache its rendered value.
      * <p>
      * Use {@link #ofValue(Object)} to build a property with a value instead.
      */
@@ -151,15 +129,15 @@ public class Property<T> {
             throw new IllegalArgumentException("'expression' must be a valid Pebble expression");
         }
 
-        return new Property<>(expression, true);
+        return new Property<>(expression);
     }
 
     /**
-     * Render a property, then convert it to its target type.<br>
+     * Render a property then convert it to its target type.<br>
      * <p>
-     * This method is designed to be used only by the {@link RunContextProperty}.
+     * This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
      *
-     * @see RunContextProperty#as(Class)
+     * @see io.kestra.core.runners.RunContextProperty#as(Class)
      */
     public static <T> T as(Property<T> property, PropertyContext context, Class<T> clazz) throws IllegalVariableEvaluationException {
         return as(property, context, clazz, Map.of());
@@ -168,57 +146,25 @@ public class Property<T> {
     /**
      * Render a property with additional variables, then convert it to its target type.<br>
      * <p>
-     * This method is designed to be used only by the {@link RunContextProperty}.
+     * This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
      *
-     * @see RunContextProperty#as(Class, Map)
+     * @see io.kestra.core.runners.RunContextProperty#as(Class, Map)
      */
     public static <T> T as(Property<T> property, PropertyContext context, Class<T> clazz, Map<String, Object> variables) throws IllegalVariableEvaluationException {
-        if (property.skipCache || property.value == null) {
+        if (property.value == null) {
             String rendered = context.render(property.expression, variables);
-            property.value = deserialize(rendered, clazz);
+            property.value = MAPPER.convertValue(rendered, clazz);
         }
 
         return property.value;
     }
 
-    private static <T> T deserialize(Object rendered, Class<T> clazz) throws IllegalVariableEvaluationException {
-        try {
-            return MAPPER.convertValue(rendered, clazz);
-        } catch (IllegalArgumentException e) {
-            if (rendered instanceof String str) {
-                try {
-                    return MAPPER.readValue(str, clazz);
-                } catch (JsonProcessingException ex) {
-                    throw new IllegalVariableEvaluationException(ex);
-                }
-            }
-
-            throw new IllegalVariableEvaluationException(e);
-        }
-    }
-
-    private static <T> T deserialize(Object rendered, JavaType type) throws IllegalVariableEvaluationException {
-        try {
-            return MAPPER.convertValue(rendered, type);
-        } catch (IllegalArgumentException e) {
-            if (rendered instanceof String str) {
-                try {
-                    return MAPPER.readValue(str, type);
-                } catch (JsonProcessingException ex) {
-                    throw new IllegalVariableEvaluationException(ex);
-                }
-            }
-
-            throw new IllegalVariableEvaluationException(e);
-        }
-    }
-
     /**
      * Render a property then convert it as a list of target type.<br>
      * <p>
-     * This method is designed to be used only by the {@link RunContextProperty}.
+     * This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
      *
-     * @see RunContextProperty#asList(Class)
+     * @see io.kestra.core.runners.RunContextProperty#asList(Class)
      */
     public static <T, I> T asList(Property<T> property, PropertyContext context, Class<I> itemClazz) throws IllegalVariableEvaluationException {
         return asList(property, context, itemClazz, Map.of());
@@ -227,39 +173,37 @@ public class Property<T> {
     /**
      * Render a property with additional variables, then convert it as a list of target type.<br>
      * <p>
-     * This method is designed to be used only by the {@link RunContextProperty}.
+     * This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
      *
-     * @see RunContextProperty#asList(Class, Map)
+     * @see io.kestra.core.runners.RunContextProperty#asList(Class, Map)
      */
     @SuppressWarnings("unchecked")
     public static <T, I> T asList(Property<T> property, PropertyContext context, Class<I> itemClazz, Map<String, Object> variables) throws IllegalVariableEvaluationException {
-        if (property.skipCache || property.value == null) {
+        if (property.value == null) {
             JavaType type = MAPPER.getTypeFactory().constructCollectionLikeType(List.class, itemClazz);
-            String trimmedExpression = property.expression.trim();
-            // We need to detect if the expression is already a list or if it's a pebble expression (for eg. referencing a variable containing a list).
-            // Doing that allows us to, if it's an expression, first render then read it as a list.
-            if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
-                property.value = deserialize(context.render(property.expression, variables), type);
-            }
-            // Otherwise, if it's already a list, we read it as a list first then render it from run context which handle list rendering by rendering each item of the list
-            else {
-                List<?> asRawList = deserialize(property.expression, List.class);
-                property.value = (T) asRawList.stream()
-                    .map(throwFunction(item -> {
-                        Object rendered = null;
-                        if (item instanceof String str) {
-                            rendered = context.render(str, variables);
-                        } else if (item instanceof Map map) {
-                            rendered = context.render(map, variables);
-                        }
-
-                        if (rendered != null) {
-                            return deserialize(rendered, itemClazz);
-                        }
-
-                        return item;
-                    }))
-                    .toList();
+            try {
+                String trimmedExpression = property.expression.trim();
+                // We need to detect if the expression is already a list or if it's a pebble expression (for eg. referencing a variable containing a list).
+                // Doing that allows us to, if it's an expression, first render then read it as a list.
+                if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
+                    property.value = MAPPER.readValue(context.render(property.expression, variables), type);
+                }
+                // Otherwise, if it's already a list, we read it as a list first then render it from run context which handle list rendering by rendering each item of the list
+                else {
+                    List<?> asRawList = MAPPER.readValue(property.expression, List.class);
+                    property.value = (T) asRawList.stream()
+                        .map(throwFunction(item -> {
+                            if (item instanceof String str) {
+                                return MAPPER.convertValue(context.render(str, variables), itemClazz);
+                            } else if (item instanceof Map map) {
+                                return MAPPER.convertValue(context.render(map, variables), itemClazz);
+                            }
+                            return item;
+                        }))
+                        .toList();
+                }
+            } catch (JsonProcessingException e) {
+                throw new IllegalVariableEvaluationException(e);
             }
         }
 
@@ -269,9 +213,9 @@ public class Property<T> {
     /**
      * Render a property then convert it as a map of target types.<br>
      * <p>
-     * This method is designed to be used only by the {@link RunContextProperty}.
+     * This method is designed to be used only by the {@link io.kestra.core.runners.RunContextProperty}.
      *
-     * @see RunContextProperty#asMap(Class, Class)
+     * @see io.kestra.core.runners.RunContextProperty#asMap(Class, Class)
      */
     public static <T, K, V> T asMap(Property<T> property, RunContext runContext, Class<K> keyClass, Class<V> valueClass) throws IllegalVariableEvaluationException {
         return asMap(property, runContext, keyClass, valueClass, Map.of());
@@ -283,11 +227,11 @@ public class Property<T> {
      * This method is safe to be used as many times as you want as the rendering and conversion will be cached.
      * Warning, due to the caching mechanism, this method is not thread-safe.
      *
-     * @see RunContextProperty#asMap(Class, Class, Map)
+     * @see io.kestra.core.runners.RunContextProperty#asMap(Class, Class, Map)
      */
     @SuppressWarnings({"rawtypes", "unchecked"})
     public static <T, K, V> T asMap(Property<T> property, RunContext runContext, Class<K> keyClass, Class<V> valueClass, Map<String, Object> variables) throws IllegalVariableEvaluationException {
-        if (property.skipCache || property.value == null) {
+        if (property.value == null) {
             JavaType targetMapType = MAPPER.getTypeFactory().constructMapType(Map.class, keyClass, valueClass);
 
             try {
@@ -295,12 +239,12 @@ public class Property<T> {
                 // We need to detect if the expression is already a map or if it's a pebble expression (for eg. referencing a variable containing a map).
                 // Doing that allows us to, if it's an expression, first render then read it as a map.
                 if (trimmedExpression.startsWith("{{") && trimmedExpression.endsWith("}}")) {
-                    property.value = deserialize(runContext.render(property.expression, variables), targetMapType);
+                    property.value = MAPPER.readValue(runContext.render(property.expression, variables), targetMapType);
                 }
                 // Otherwise if it's already a map we read it as a map first then render it from run context which handle map rendering by rendering each entry of the map (otherwise it will fail with nested expressions in values for eg.)
                 else {
                     Map asRawMap = MAPPER.readValue(property.expression, Map.class);
-                    property.value = deserialize(runContext.render(asRawMap, variables), targetMapType);
+                    property.value = MAPPER.convertValue(runContext.render(asRawMap, variables), targetMapType);
                 }
             } catch (JsonProcessingException e) {
                 throw new IllegalVariableEvaluationException(e);

core/src/main/java/io/kestra/core/models/tasks/runners/TaskException.java

@@ -15,31 +15,10 @@ public class TaskException extends Exception {
 
     private transient AbstractLogConsumer logConsumer;
 
-    /**
-     * This constructor will certainly be removed in 0.21 as we keep it only because all task runners must be impacted.
-     * @deprecated use {@link #TaskException(int, AbstractLogConsumer)} instead.
-     */
-    @Deprecated(forRemoval = true, since = "0.20.0")
-    public TaskException(int exitCode, int stdOutCount, int stdErrCount) {
-        this("Command failed with exit code " + exitCode, exitCode, stdOutCount, stdErrCount);
-    }
-
     public TaskException(int exitCode, AbstractLogConsumer logConsumer) {
         this("Command failed with exit code " + exitCode, exitCode, logConsumer);
     }
 
-    /**
-     * This constructor will certainly be removed in 0.21 as we keep it only because all task runners must be impacted.
-     * @deprecated use {@link #TaskException(String, int, AbstractLogConsumer)} instead.
-     */
-    @Deprecated(forRemoval = true, since = "0.20.0")
-    public TaskException(String message, int exitCode, int stdOutCount, int stdErrCount) {
-        super(message);
-        this.exitCode = exitCode;
-        this.stdOutCount = stdOutCount;
-        this.stdErrCount = stdErrCount;
-    }
-
     public TaskException(String message, int exitCode, AbstractLogConsumer logConsumer) {
         super(message);
         this.exitCode = exitCode;

core/src/main/java/io/kestra/core/models/templates/Template.java

@@ -1,156 +0,0 @@
-package io.kestra.core.models.templates;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.introspect.AnnotatedMember;
-import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
-import io.kestra.core.models.DeletedInterface;
-import io.kestra.core.models.HasUID;
-import io.kestra.core.models.TenantInterface;
-import io.kestra.core.models.tasks.Task;
-import io.kestra.core.models.validations.ManualConstraintViolation;
-import io.kestra.core.serializers.JacksonMapper;
-import io.kestra.core.utils.IdUtils;
-import io.micronaut.core.annotation.Introspected;
-import io.swagger.v3.oas.annotations.Hidden;
-import lombok.*;
-import lombok.experimental.SuperBuilder;
-
-import java.util.*;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.ConstraintViolationException;
-import jakarta.validation.Valid;
-import jakarta.validation.constraints.NotBlank;
-import jakarta.validation.constraints.NotEmpty;
-import jakarta.validation.constraints.NotNull;
-import jakarta.validation.constraints.Pattern;
-
-@SuperBuilder(toBuilder = true)
-@Getter
-@AllArgsConstructor
-@NoArgsConstructor
-@Introspected
-@ToString
-@EqualsAndHashCode
-public class Template implements DeletedInterface, TenantInterface, HasUID {
-    private static final ObjectMapper YAML_MAPPER = JacksonMapper.ofYaml().copy()
-        .setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
-            @Override
-            public boolean hasIgnoreMarker(final AnnotatedMember m) {
-                List<String> exclusions = Arrays.asList("revision", "deleted", "source");
-                return exclusions.contains(m.getName()) || super.hasIgnoreMarker(m);
-            }
-        })
-        .setDefaultPropertyInclusion(JsonInclude.Include.NON_DEFAULT);
-
-    @Setter
-    @Hidden
-    @Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
-    private String tenantId;
-
-    @NotNull
-    @NotBlank
-    @Pattern(regexp = "^[a-zA-Z0-9][a-zA-Z0-9._-]*")
-    private String id;
-
-    @NotNull
-    @Pattern(regexp="^[a-z0-9][a-z0-9._-]*")
-    private String namespace;
-
-    String description;
-
-    @Valid
-    @NotEmpty
-    private List<Task> tasks;
-
-    @Valid
-    private List<Task> errors;
-
-    @Valid
-    @JsonProperty("finally")
-    @Getter(AccessLevel.NONE)
-    protected List<Task> _finally;
-
-    public List<Task> getFinally() {
-        return this._finally;
-    }
-
-    @NotNull
-    @Builder.Default
-    private final boolean deleted = false;
-
-
-    /** {@inheritDoc **/
-    @Override
-    @JsonIgnore
-    public String uid() {
-        return Template.uid(
-            this.getTenantId(),
-            this.getNamespace(),
-            this.getId()
-        );
-    }
-
-    @JsonIgnore
-    public static String uid(String tenantId, String namespace, String id) {
-        return IdUtils.fromParts(
-            tenantId,
-            namespace,
-            id
-        );
-    }
-
-    public Optional<ConstraintViolationException> validateUpdate(Template updated) {
-        Set<ConstraintViolation<?>> violations = new HashSet<>();
-
-        if (!updated.getId().equals(this.getId())) {
-            violations.add(ManualConstraintViolation.of(
-                "Illegal template id update",
-                updated,
-                Template.class,
-                "template.id",
-                updated.getId()
-            ));
-        }
-
-        if (!updated.getNamespace().equals(this.getNamespace())) {
-            violations.add(ManualConstraintViolation.of(
-                "Illegal namespace update",
-                updated,
-                Template.class,
-                "template.namespace",
-                updated.getNamespace()
-            ));
-        }
-
-        if (!violations.isEmpty()) {
-            return Optional.of(new ConstraintViolationException(violations));
-        } else {
-            return Optional.empty();
-        }
-    }
-
-    public String generateSource() {
-        try {
-            return YAML_MAPPER.writeValueAsString(this);
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public Template toDeleted() {
-        return new Template(
-            this.tenantId,
-            this.id,
-            this.namespace,
-            this.description,
-            this.tasks,
-            this.errors,
-            this._finally,
-            true
-        );
-    }
-}

core/src/main/java/io/kestra/core/models/templates/TemplateEnabled.java

@@ -1,15 +0,0 @@
-package io.kestra.core.models.templates;
-
-import io.micronaut.context.annotation.Requires;
-import io.micronaut.core.util.StringUtils;
-
-import java.lang.annotation.*;
-
-@Documented
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.PACKAGE, ElementType.TYPE})
-@Requires(property = "kestra.templates.enabled", value = StringUtils.TRUE, defaultValue = StringUtils.FALSE)
-@Inherited
-public @interface TemplateEnabled {
-
-}

core/src/main/java/io/kestra/core/models/templates/TemplateSource.java

@@ -1,18 +0,0 @@
-package io.kestra.core.models.templates;
-
-import io.micronaut.core.annotation.Introspected;
-import lombok.*;
-import lombok.experimental.SuperBuilder;
-import lombok.extern.jackson.Jacksonized;
-
-@SuperBuilder
-@Getter
-@AllArgsConstructor
-@NoArgsConstructor
-@Introspected
-@ToString
-@EqualsAndHashCode
-public class TemplateSource extends Template {
-    String source;
-    String exception;
-}

core/src/main/java/io/kestra/core/plugins/notifications/ExecutionInterface.java

@@ -1,25 +0,0 @@
-package io.kestra.core.plugins.notifications;
-
-import io.kestra.core.models.property.Property;
-import io.swagger.v3.oas.annotations.media.Schema;
-
-import java.util.Map;
-
-public interface ExecutionInterface {
-    @Schema(
-        title = "The execution id to use",
-        description = "Default is the current execution, " +
-            "change it to {{ trigger.executionId }} if you use this task with a Flow trigger to use the original execution."
-    )
-    Property<String> getExecutionId();
-
-    @Schema(
-        title = "Custom fields to be added on notification"
-    )
-    Property<Map<String, Object>> getCustomFields();
-
-    @Schema(
-        title = "Custom message to be added on notification"
-    )
-    Property<String> getCustomMessage();
-}

core/src/main/java/io/kestra/core/plugins/notifications/ExecutionService.java

@@ -1,141 +0,0 @@
-package io.kestra.core.plugins.notifications;
-
-import io.kestra.core.exceptions.IllegalVariableEvaluationException;
-import io.kestra.core.models.executions.Execution;
-import io.kestra.core.models.executions.TaskRun;
-import io.kestra.core.models.flows.State;
-import io.kestra.core.models.property.Property;
-import io.kestra.core.models.tasks.retrys.Exponential;
-import io.kestra.core.repositories.ExecutionRepositoryInterface;
-import io.kestra.core.runners.DefaultRunContext;
-import io.kestra.core.runners.RunContext;
-import io.kestra.core.serializers.JacksonMapper;
-import io.kestra.core.utils.ListUtils;
-import io.kestra.core.utils.RetryUtils;
-import io.kestra.core.utils.UriProvider;
-
-import java.time.Duration;
-import java.util.*;
-
-public final class ExecutionService {
-    private ExecutionService() {}
-
-    public static Execution findExecution(RunContext runContext, Property<String> executionId) throws IllegalVariableEvaluationException, NoSuchElementException {
-        ExecutionRepositoryInterface executionRepository = ((DefaultRunContext) runContext).getApplicationContext().getBean(ExecutionRepositoryInterface.class);
-        RetryUtils retryUtils = ((DefaultRunContext) runContext).getApplicationContext().getBean(RetryUtils.class);
-
-        RetryUtils.Instance<Execution, NoSuchElementException> retryInstance = retryUtils
-            .of(Exponential.builder()
-                    .delayFactor(2.0)
-                    .interval(Duration.ofSeconds(1))
-                    .maxInterval(Duration.ofSeconds(15))
-                    .maxAttempts(-1)
-                    .maxDuration(Duration.ofMinutes(10))
-                    .build(),
-                runContext.logger()
-            );
-
-        var executionRendererId = runContext.render(executionId).as(String.class).orElse(null);
-        var flowTriggerExecutionState = getOptionalFlowTriggerExecutionState(runContext);
-
-        var flowVars = (Map<String, String>) runContext.getVariables().get("flow");
-        var isCurrentExecution = isCurrentExecution(runContext, executionRendererId);
-        if (isCurrentExecution) {
-            runContext.logger().info("Loading execution data for the current execution.");
-        }
-
-        return retryInstance.run(
-            NoSuchElementException.class,
-            () -> executionRepository.findById(flowVars.get("tenantId"), executionRendererId)
-                .filter(foundExecution -> isExecutionInTheWantedState(foundExecution, isCurrentExecution, flowTriggerExecutionState))
-                .orElseThrow(() -> new NoSuchElementException("Unable to find execution '" + executionRendererId + "'"))
-
-        );
-    }
-
-    /**
-     * ExecutionRepository can be out of sync in ElasticSearch stack, with this filter we try to mitigate that
-     *
-     * @param execution                 the Execution we fetched from ExecutionRepository
-     * @param isCurrentExecution        true if this *Execution Task is configured to send a notification for the current Execution
-     * @param flowTriggerExecutionState the Execution State that triggered the Flow trigger, if any
-     * @return true if we think we fetched the right Execution data for our usecase
-     */
-    public static boolean isExecutionInTheWantedState(Execution execution, boolean isCurrentExecution, Optional<String> flowTriggerExecutionState) {
-        if (isCurrentExecution) {
-            // we don't wait for current execution to be terminated as it could not be possible as long as this task is running
-            return true;
-        }
-
-        if (flowTriggerExecutionState.isPresent()) {
-            // we were triggered by a Flow trigger that can be, for example: PAUSED
-            if (flowTriggerExecutionState.get().equals(State.Type.RUNNING.toString())) {
-                // RUNNING special case: we take the first state we got
-                return true;
-            } else {
-                // to handle the case where the ExecutionRepository is out of sync in ElasticSearch stack,
-                // we try to match an Execution with the same state
-                return execution.getState().getCurrent().name().equals(flowTriggerExecutionState.get());
-            }
-        } else {
-            return execution.getState().getCurrent().isTerminated();
-        }
-    }
-
-    public static Map<String, Object> executionMap(RunContext runContext, ExecutionInterface executionInterface) throws IllegalVariableEvaluationException {
-        Execution execution = findExecution(runContext, executionInterface.getExecutionId());
-        UriProvider uriProvider = ((DefaultRunContext) runContext).getApplicationContext().getBean(UriProvider.class);
-
-        Map<String, Object> templateRenderMap = new HashMap<>();
-        templateRenderMap.put("duration", execution.getState().humanDuration());
-        templateRenderMap.put("startDate", execution.getState().getStartDate());
-        templateRenderMap.put("link", uriProvider.executionUrl(execution));
-        templateRenderMap.put("execution", JacksonMapper.toMap(execution));
-
-        runContext.render(executionInterface.getCustomMessage())
-            .as(String.class)
-            .ifPresent(s -> templateRenderMap.put("customMessage", s));
-
-        final Map<String, Object> renderedCustomFields = runContext.render(executionInterface.getCustomFields()).asMap(String.class, Object.class);
-        if (!renderedCustomFields.isEmpty()) {
-            templateRenderMap.put("customFields", renderedCustomFields);
-        }
-
-        var isCurrentExecution = isCurrentExecution(runContext, execution.getId());
-
-        List<TaskRun> taskRuns;
-
-        if (isCurrentExecution) {
-            taskRuns = execution.getTaskRunList();
-        } else {
-            taskRuns = execution.getTaskRunList().stream()
-                .filter(t -> (execution.hasFailed() ? State.Type.FAILED : State.Type.SUCCESS).equals(t.getState().getCurrent()))
-                .toList();
-        }
-
-        if (!ListUtils.isEmpty(taskRuns)) {
-            TaskRun lastTaskRun = taskRuns.getLast();
-            templateRenderMap.put("firstFailed", State.Type.FAILED.equals(lastTaskRun.getState().getCurrent()) ? lastTaskRun : false);
-            templateRenderMap.put("lastTask", lastTaskRun);
-        }
-
-        return templateRenderMap;
-    }
-
-    /**
-     * if there is a state, we assume this is a Flow trigger with type: {@link io.kestra.plugin.core.trigger.Flow.Output}
-     *
-     * @return the state of the execution that triggered the Flow trigger, or empty if another usecase/trigger
-     */
-    private static Optional<String> getOptionalFlowTriggerExecutionState(RunContext runContext) {
-        var triggerVar = Optional.ofNullable(
-            runContext.getVariables().get("trigger")
-        );
-        return triggerVar.map(trigger -> ((Map<String, String>) trigger).get("state"));
-    }
-
-    private static boolean isCurrentExecution(RunContext runContext, String executionId) {
-        var executionVars = (Map<String, String>) runContext.getVariables().get("execution");
-        return executionId.equals(executionVars.get("id"));
-    }
-}

core/src/main/java/io/kestra/core/queues/QueueFactoryInterface.java

@@ -5,22 +5,19 @@ import io.kestra.core.models.executions.ExecutionKilled;
 import io.kestra.core.models.executions.LogEntry;
 import io.kestra.core.models.executions.MetricEntry;
 import io.kestra.core.models.flows.FlowInterface;
-import io.kestra.core.models.templates.Template;
 import io.kestra.core.models.triggers.Trigger;
 import io.kestra.core.runners.*;
 
 public interface QueueFactoryInterface {
     String EXECUTION_NAMED = "executionQueue";
-    String EXECUTOR_NAMED = "executorQueue";
+    String EXECUTION_EVENT_NAMED = "executionEventQueue";
     String WORKERJOB_NAMED = "workerJobQueue";
     String WORKERTASKRESULT_NAMED = "workerTaskResultQueue";
     String WORKERTRIGGERRESULT_NAMED = "workerTriggerResultQueue";
     String FLOW_NAMED = "flowQueue";
-    String TEMPLATE_NAMED = "templateQueue";
     String WORKERTASKLOG_NAMED = "workerTaskLogQueue";
     String METRIC_QUEUE = "workerTaskMetricQueue";
     String KILL_NAMED = "executionKilledQueue";
-    String WORKERINSTANCE_NAMED = "workerInstanceQueue";
     String WORKERJOBRUNNING_NAMED = "workerJobRunningQueue";
     String TRIGGER_NAMED = "triggerQueue";
     String SUBFLOWEXECUTIONRESULT_NAMED = "subflowExecutionResultQueue";
@@ -30,7 +27,7 @@ public interface QueueFactoryInterface {
 
     QueueInterface<Execution> execution();
 
-    QueueInterface<Executor> executor();
+    QueueInterface<ExecutionEvent> executionEvent();
 
     WorkerJobQueueInterface workerJob();
 
@@ -46,10 +43,6 @@ public interface QueueFactoryInterface {
 
     QueueInterface<ExecutionKilled> kill();
 
-    QueueInterface<Template> template();
-
-    QueueInterface<WorkerInstance> workerInstance();
-
     QueueInterface<WorkerJobRunning> workerJobRunning();
 
     QueueInterface<Trigger> trigger();

core/src/main/java/io/kestra/core/queues/QueueInterface.java

@@ -35,6 +35,24 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
 
     void delete(String consumerGroup, T message) throws QueueException;
 
+    /**
+     * Delete all messages of the queue for this key.
+     * This is used to purge a queue for a specific key.
+     * A queue implementation may omit to implement it and purge records differently.
+     */
+    default void deleteByKey(String key) throws QueueException {
+        // by default do nothing
+    }
+
+    /**
+     * Delete all messages of the queue for a set of keys.
+     * This is used to purge a queue for specific keys.
+     * A queue implementation may omit to implement it and purge records differently.
+     */
+    default void deleteByKeys(List<String> keys) throws QueueException {
+        // by default do nothing
+    }
+
     default Runnable receive(Consumer<Either<T, DeserializationException>> consumer) {
         return receive(null, consumer, false);
     }
@@ -54,4 +72,20 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
     }
 
     Runnable receive(String consumerGroup, Class<?> queueType, Consumer<Either<T, DeserializationException>> consumer, boolean forUpdate);
+
+    default Runnable receiveBatch(Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
+        return receiveBatch(null, queueType, consumer);
+    }
+
+    default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
+        return receiveBatch(consumerGroup, queueType, consumer, true);
+    }
+
+    /**
+     * Consumer a batch of messages.
+     * By default, it consumes a single message, a queue implementation may implement it to support batch consumption.
+     */
+    default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer, boolean forUpdate) {
+        return receive(consumerGroup, either -> consumer.accept(List.of(either)), forUpdate);
+    }
 }

core/src/main/java/io/kestra/core/queues/QueueService.java

@@ -19,12 +19,8 @@ public class QueueService {
             return ((SubflowExecution<?>) object).getExecution().getId();
         } else if (object.getClass() == SubflowExecutionResult.class) {
             return ((SubflowExecutionResult) object).getExecutionId();
-        } else if (object.getClass() == ExecutorState.class) {
-            return ((ExecutorState) object).getExecutionId();
         } else if (object.getClass() == Setting.class) {
             return ((Setting) object).getKey();
-        } else if (object.getClass() == Executor.class) {
-            return ((Executor) object).getExecution().getId();
         } else if (object.getClass() == MetricEntry.class) {
             return null;
         } else if (object.getClass() == SubflowExecutionEnd.class) {

core/src/main/java/io/kestra/core/repositories/ConcurrencyLimitRepositoryInterface.java

@@ -0,0 +1,25 @@
+package io.kestra.core.repositories;
+
+import io.kestra.core.runners.ConcurrencyLimit;
+import jakarta.validation.constraints.NotNull;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface ConcurrencyLimitRepositoryInterface {
+    /**
+     * Update a concurrency limit
+     * WARNING: this is inherently unsafe and must only be used for administration
+     */
+    ConcurrencyLimit update(ConcurrencyLimit concurrencyLimit);
+
+    /**
+     * Returns all concurrency limits from the database for a given tenant
+     */
+    List<ConcurrencyLimit> find(String tenantId);
+
+    /**
+     * Find a concurrency limit by its id
+     */
+    Optional<ConcurrencyLimit> findById(@NotNull String tenantId, @NotNull String namespace, @NotNull String flowId);
+}

core/src/main/java/io/kestra/core/repositories/ExecutionRepositoryInterface.java

@@ -2,7 +2,6 @@ package io.kestra.core.repositories;
 
 import io.kestra.core.models.QueryFilter;
 import io.kestra.core.models.executions.Execution;
-import io.kestra.core.models.executions.TaskRun;
 import io.kestra.core.models.executions.statistics.DailyExecutionStatistics;
 import io.kestra.core.models.executions.statistics.ExecutionCount;
 import io.kestra.core.models.executions.statistics.Flow;

core/src/main/java/io/kestra/core/repositories/FlowTopologyRepositoryInterface.java

@@ -1,6 +1,7 @@
 package io.kestra.core.repositories;
 
 
+import io.kestra.core.models.flows.FlowInterface;
 import io.kestra.core.models.topologies.FlowTopology;
 
 import java.util.List;
@@ -13,4 +14,6 @@ public interface FlowTopologyRepositoryInterface {
     List<FlowTopology> findAll(String tenantId);
 
     FlowTopology save(FlowTopology flowTopology);
+
+    void save(FlowInterface flow, List<FlowTopology> flowTopologies);
 }

core/src/main/java/io/kestra/core/repositories/LockRepositoryInterface.java

@@ -0,0 +1,24 @@
+package io.kestra.core.repositories;
+
+import io.kestra.core.lock.Lock;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Low lever repository for locks.
+ * It should never be used directly but only via the {@link io.kestra.core.lock.LockService}.
+ */
+public interface LockRepositoryInterface {
+    Optional<Lock> findById(String category, String id);
+
+    boolean create(Lock newLock);
+
+    default void delete(Lock existing) {
+        deleteById(existing.getCategory(), existing.getId());
+    }
+
+    void deleteById(String category, String id);
+
+    List<Lock> deleteByOwner(String owner);
+}

core/src/main/java/io/kestra/core/repositories/SaveRepositoryInterface.java

@@ -5,7 +5,5 @@ import java.util.List;
 public interface SaveRepositoryInterface<T> {
     T save(T item);
 
-    default int saveBatch(List<T> items) {
-        throw new UnsupportedOperationException();
-    }
+    int saveBatch(List<T> items);
 }

core/src/main/java/io/kestra/core/repositories/ServiceInstanceRepositoryInterface.java

@@ -1,7 +1,9 @@
 package io.kestra.core.repositories;
 
+import io.kestra.core.runners.TransactionContext;
 import io.kestra.core.server.Service;
 import io.kestra.core.server.ServiceInstance;
+import io.kestra.core.server.ServiceStateTransition;
 import io.kestra.core.server.ServiceType;
 import io.micronaut.data.model.Pageable;
 
@@ -9,6 +11,7 @@ import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 
 /**
@@ -58,20 +61,6 @@ public interface ServiceInstanceRepositoryInterface {
      */
     ServiceInstance save(ServiceInstance service);
 
-    /**
-     * Finds all service instances which are in the given state.
-     *
-     * @return the list of {@link ServiceInstance}.
-     */
-    List<ServiceInstance> findAllInstancesInState(final Service.ServiceState state);
-
-    /**
-     * Finds all service instances which are in the given state.
-     *
-     * @return the list of {@link ServiceInstance}.
-     */
-    List<ServiceInstance> findAllInstancesInStates(final Set<Service.ServiceState> states);
-
     /**
      * Finds all service active instances between the given dates.
      *
@@ -84,6 +73,28 @@ public interface ServiceInstanceRepositoryInterface {
                                                   final Instant from,
                                                   final Instant to);
 
+    /**
+     * Finds all service instances which are NOT {@link Service.ServiceState#RUNNING}, then process them using the consumer.
+     */
+    void processAllNonRunningInstances(BiConsumer<TransactionContext, ServiceInstance> consumer);
+
+    /**
+     * Attempt to transition the state of a given service to a given new state.
+     * This method may not update the service if the transition is not valid.
+     *
+     * @param instance the service instance.
+     * @param newState the new state of the service.
+     * @return an optional of the {@link ServiceInstance} or {@link Optional#empty()} if the service is not running.
+     */
+    ServiceStateTransition.Response mayTransitServiceTo(final TransactionContext txContext,
+                                                        final ServiceInstance instance,
+                                                        final Service.ServiceState newState,
+                                                        final String reason);
+
+    /**
+     * Finds all service instances that are in the states, then process them using the consumer.
+     */
+    void processInstanceInStates(Set<Service.ServiceState> states, BiConsumer<TransactionContext, ServiceInstance> consumer);
     /**
      * Purge all instances in the EMPTY state older than the until date.
      *

core/src/main/java/io/kestra/core/repositories/TemplateRepositoryInterface.java

@@ -1,42 +0,0 @@
-package io.kestra.core.repositories;
-
-import io.micronaut.data.model.Pageable;
-import io.kestra.core.models.templates.Template;
-
-import java.util.List;
-import java.util.Optional;
-import jakarta.annotation.Nullable;
-
-public interface TemplateRepositoryInterface {
-    Optional<Template> findById(String tenantId, String namespace, String id);
-
-    List<Template> findAll(String tenantId);
-
-    List<Template> findAllWithNoAcl(String tenantId);
-
-    List<Template> findAllForAllTenants();
-
-    ArrayListTotal<Template> find(
-        Pageable pageable,
-        @Nullable String query,
-        @Nullable String tenantId,
-        @Nullable String namespace
-    );
-
-    // Should normally be TemplateWithSource but it didn't exist yet
-    List<Template> find(
-        @Nullable String query,
-        @Nullable String tenantId,
-        @Nullable String namespace
-    );
-
-    List<Template> findByNamespace(String tenantId, String namespace);
-
-    Template create(Template template);
-
-    Template update(Template template, Template previous);
-
-    void delete(Template template);
-
-    List<String> findDistinctNamespace(String tenantId);
-}

core/src/main/java/io/kestra/core/repositories/WorkerJobRunningRepositoryInterface.java

@@ -1,13 +0,0 @@
-package io.kestra.core.repositories;
-
-
-import io.kestra.core.runners.WorkerJobRunning;
-
-import java.util.Optional;
-
-public interface WorkerJobRunningRepositoryInterface {
-    Optional<WorkerJobRunning> findByKey(String uid);
-
-    void deleteByKey(String uid);
-
-}

core/src/main/java/io/kestra/core/runners/DefaultFlowMetaStore.java

@@ -1,9 +1,11 @@
 package io.kestra.core.runners;
 
+import io.kestra.core.models.executions.Execution;
 import io.kestra.core.models.flows.FlowInterface;
 import io.kestra.core.models.flows.FlowWithSource;
 import io.kestra.core.repositories.FlowRepositoryInterface;
 import io.kestra.core.services.FlowListenersInterface;
+import io.kestra.core.services.PluginDefaultService;
 import jakarta.inject.Singleton;
 import java.util.Objects;
 import lombok.Setter;
@@ -15,12 +17,14 @@ import java.util.Optional;
 @Singleton
 public class DefaultFlowMetaStore implements FlowMetaStoreInterface {
     private final FlowRepositoryInterface flowRepository;
+    private final PluginDefaultService pluginDefaultService;
 
     @Setter
     private List<FlowWithSource> allFlows;
 
-    public DefaultFlowMetaStore(FlowListenersInterface flowListeners, FlowRepositoryInterface flowRepository) {
+    public DefaultFlowMetaStore(FlowListenersInterface flowListeners, FlowRepositoryInterface flowRepository, PluginDefaultService pluginDefaultService) {
         this.flowRepository = flowRepository;
+        this.pluginDefaultService = pluginDefaultService;
         flowListeners.listen(flows -> allFlows = flows);
     }
 
@@ -53,4 +57,9 @@ public class DefaultFlowMetaStore implements FlowMetaStoreInterface {
     public Boolean isReady() {
         return true;
     }
+
+    @Override
+    public Optional<FlowWithSource> findByExecutionThenInjectDefaults(Execution execution) {
+        return findByExecution(execution).map(it -> pluginDefaultService.injectDefaults(it, execution));
+    }
 }

core/src/main/java/io/kestra/core/runners/DefaultIndexer.java

@@ -1,4 +1,4 @@
-package io.kestra.jdbc.runner;
+package io.kestra.core.runners;
 
 import io.kestra.core.metrics.MetricRegistry;
 import io.kestra.core.models.executions.LogEntry;
@@ -9,7 +9,6 @@ import io.kestra.core.queues.QueueService;
 import io.kestra.core.repositories.LogRepositoryInterface;
 import io.kestra.core.repositories.MetricRepositoryInterface;
 import io.kestra.core.repositories.SaveRepositoryInterface;
-import io.kestra.core.runners.Indexer;
 import io.kestra.core.server.ServiceStateChangeEvent;
 import io.kestra.core.server.ServiceType;
 import io.kestra.core.utils.IdUtils;
@@ -29,20 +28,20 @@ import jakarta.inject.Named;
 import jakarta.inject.Singleton;
 import lombok.extern.slf4j.Slf4j;
 
+// FIXME move that to a new indexer module
 /**
  * This class is responsible to batch-indexed asynchronously queue messages.<p>
- * Some queue messages are indexed synchronously via the {@link JdbcQueueIndexer}.
+ * Some queue messages are indexed synchronously via the {@link QueueIndexer}.
  */
 @SuppressWarnings("this-escape")
 @Slf4j
 @Singleton
-@JdbcRunnerEnabled
-public class JdbcIndexer implements Indexer {
+public class DefaultIndexer implements Indexer {
     private final LogRepositoryInterface logRepository;
-    private final JdbcQueue<LogEntry> logQueue;
+    private final QueueInterface<LogEntry> logQueue;
 
     private final MetricRepositoryInterface metricRepository;
-    private final JdbcQueue<MetricEntry> metricQueue;
+    private final QueueInterface<MetricEntry> metricQueue;
     private final MetricRegistry metricRegistry;
     private final List<Runnable> receiveCancellations = new ArrayList<>();
 
@@ -56,7 +55,7 @@ public class JdbcIndexer implements Indexer {
     private final QueueService queueService;
 
     @Inject
-    public JdbcIndexer(
+    public DefaultIndexer(
         LogRepositoryInterface logRepository,
         @Named(QueueFactoryInterface.WORKERTASKLOG_NAMED) QueueInterface<LogEntry> logQueue,
         MetricRepositoryInterface metricRepositor,
@@ -67,9 +66,9 @@ public class JdbcIndexer implements Indexer {
         QueueService queueService
     ) {
         this.logRepository = logRepository;
-        this.logQueue = (JdbcQueue<LogEntry>) logQueue;
+        this.logQueue = logQueue;
         this.metricRepository = metricRepositor;
-        this.metricQueue = (JdbcQueue<MetricEntry>) metricQueue;
+        this.metricQueue = metricQueue;
         this.metricRegistry = metricRegistry;
         this.eventPublisher = eventPublisher;
         this.skipExecutionService = skipExecutionService;
@@ -91,7 +90,7 @@ public class JdbcIndexer implements Indexer {
         this.sendBatch(metricQueue, metricRepository);
     }
 
-    protected <T> void sendBatch(JdbcQueue<T> queueInterface, SaveRepositoryInterface<T> saveRepositoryInterface) {
+    protected <T> void sendBatch(QueueInterface<T> queueInterface, SaveRepositoryInterface<T> saveRepositoryInterface) {
         this.receiveCancellations.addFirst(queueInterface.receiveBatch(Indexer.class, eithers -> {
             // first, log all deserialization issues
             eithers.stream().filter(either -> either.isRight()).forEach(either -> log.error("unable to deserialize an item: {}", either.getRight().getMessage()));

core/src/main/java/io/kestra/core/runners/DefaultQueueIndexer.java

@@ -0,0 +1,89 @@
+package io.kestra.core.runners;
+
+import io.kestra.core.metrics.MetricRegistry;
+import io.kestra.core.repositories.FlowTopologyRepositoryInterface;
+import io.micronaut.context.ApplicationContext;
+import jakarta.annotation.PostConstruct;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This class is responsible to index the queue synchronously at message production time. It is used by the Queue itself<p>
+ * Some queue messages are batch-indexed asynchronously via the regular {@link io.kestra.core.runners.Indexer}
+ * which listen to (receive) those queue messages.
+ */
+@Slf4j
+@Singleton
+public class DefaultQueueIndexer implements QueueIndexer {
+    private volatile Map<Class<?>, QueueIndexerRepository<?>> repositories;
+
+    private final MetricRegistry metricRegistry;
+    private final ApplicationContext applicationContext;
+
+    @Inject
+    public DefaultQueueIndexer(ApplicationContext applicationContext) {
+        this.applicationContext = applicationContext;
+        this.metricRegistry = applicationContext.getBean(MetricRegistry.class);
+    }
+
+    private Map<Class<?>, QueueIndexerRepository<?>> getRepositories() {
+        if (repositories == null) {
+            synchronized (this) {
+                if (repositories == null) {
+                    repositories = new HashMap<>();
+                    applicationContext.getBeansOfType(QueueIndexerRepository.class)
+                        .forEach(saveRepositoryInterface -> {
+                            repositories.put(saveRepositoryInterface.getItemClass(), saveRepositoryInterface);
+                        });
+                }
+            }
+        }
+
+        return repositories;
+    }
+
+    // FIXME this today limit this indexer to only JDBC queue and repository.
+    //  to be able to use JDBC queue with another repository we would need to check in each QueueIndexerRepository if it's a Jdbc transaction before casting
+    @Override
+    public void accept(TransactionContext txContext, Object item) {
+        Map<Class<?>, QueueIndexerRepository<?>> repositories = getRepositories();
+        if (repositories.containsKey(item.getClass())) {
+            this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_REQUEST_COUNT, MetricRegistry.METRIC_INDEXER_REQUEST_COUNT_DESCRIPTION, "type", item.getClass().getName()).increment();
+            this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_MESSAGE_IN_COUNT, MetricRegistry.METRIC_INDEXER_MESSAGE_IN_COUNT_DESCRIPTION, "type", item.getClass().getName()).increment();
+
+            this.metricRegistry.timer(MetricRegistry.METRIC_INDEXER_REQUEST_DURATION, MetricRegistry.METRIC_INDEXER_REQUEST_DURATION_DESCRIPTION, "type", item.getClass().getName()).record(() -> {
+                QueueIndexerRepository<?> indexerRepository = repositories.get(item.getClass());
+                if (indexerRepository instanceof FlowTopologyRepositoryInterface) {
+                    // we allow flow topology to fail indexation
+                    try {
+                        save(indexerRepository, txContext, item);
+                    } catch (Exception e) {
+                        log.error("Unable to index a flow topology, skipping it", e);
+                    }
+                } else {
+                    save(indexerRepository, txContext, cast(item));
+                }
+
+                this.metricRegistry.counter(MetricRegistry.METRIC_INDEXER_MESSAGE_OUT_COUNT, MetricRegistry.METRIC_INDEXER_MESSAGE_OUT_COUNT_DESCRIPTION, "type", item.getClass().getName()).increment();
+            });
+
+        }
+    }
+
+    private void save(QueueIndexerRepository<?> indexerRepository, TransactionContext txContext, Object item) {
+        if (indexerRepository.supports(txContext.getClass())) {
+            indexerRepository.save(txContext, cast(item));
+        } else {
+            indexerRepository.save(cast(item));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T cast(Object message) {
+        return (T) message;
+    }
+}

core/src/main/java/io/kestra/core/runners/DefaultRunContext.java

@@ -511,17 +511,6 @@ public class DefaultRunContext extends RunContext {
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    @SuppressWarnings("unchecked")
-    public String tenantId() {
-        Map<String, String> flow = (Map<String, String>) this.getVariables().get("flow");
-        // normally only tests should not have the flow variable
-        return flow != null ? flow.get("tenantId") : null;
-    }
-
     /**
      * {@inheritDoc}
      */

core/src/main/java/io/kestra/core/runners/ExecutableUtils.java

@@ -54,10 +54,12 @@ public final class ExecutableUtils {
     }
 
     public static SubflowExecutionResult subflowExecutionResult(TaskRun parentTaskrun, Execution execution) {
+        List<TaskRunAttempt> attempts = parentTaskrun.getAttempts() == null ? new ArrayList<>() : new ArrayList<>(parentTaskrun.getAttempts());
+        attempts.add(TaskRunAttempt.builder().state(parentTaskrun.getState()).build());
         return SubflowExecutionResult.builder()
             .executionId(execution.getId())
             .state(parentTaskrun.getState().getCurrent())
-            .parentTaskRun(parentTaskrun.addAttempt(TaskRunAttempt.builder().state(parentTaskrun.getState()).build()))
+            .parentTaskRun(parentTaskrun.withAttempts(attempts))
             .build();
     }
 

core/src/main/java/io/kestra/core/runners/ExecutionEvent.java

@@ -0,0 +1,17 @@
+package io.kestra.core.runners;
+
+import io.kestra.core.models.HasUID;
+import io.kestra.core.models.executions.Execution;
+
+import java.time.Instant;
+
+public record ExecutionEvent(String tenantId, String namespace, String flowId, String executionId, Instant eventDate, ExecutionEventType eventType) implements HasUID {
+    public ExecutionEvent(Execution execution, ExecutionEventType eventType) {
+        this(execution.getTenantId(), execution.getNamespace(), execution.getFlowId(), execution.getId(), Instant.now(), eventType);
+    }
+
+    @Override
+    public String uid() {
+        return executionId;
+    }
+}

core/src/main/java/io/kestra/core/runners/ExecutionEventType.java

@@ -0,0 +1,7 @@
+package io.kestra.core.runners;
+
+public enum ExecutionEventType {
+    CREATED,
+    UPDATED,
+    TERMINATED,
+}

core/src/main/java/io/kestra/core/runners/ExecutionQueuedStateStore.java

@@ -0,0 +1,29 @@
+package io.kestra.core.runners;
+
+import io.kestra.core.models.executions.Execution;
+import io.kestra.core.runners.ExecutionQueued;
+import io.kestra.core.runners.TransactionContext;
+
+import java.util.function.BiConsumer;
+
+/**
+ * This state store is used by the {@link Executor} to handle execution queued by flow concurrency limit.
+ */
+public interface ExecutionQueuedStateStore {
+    /**
+     * remove a queued execution.
+     */
+    void remove(Execution execution);
+
+    /**
+     * Save a queued execution.
+     *
+     * @implNote Implementors that support transaction must use the provided {@link TransactionContext} to attach to the current transaction.
+     */
+    void save(TransactionContext txContext, ExecutionQueued executionQueued);
+
+    /**
+     * Pop a queued execution: remove the oldest one and process it with the provided consumer.
+     */
+    void pop(String tenantId, String namespace, String flowId, BiConsumer<TransactionContext, Execution> consumer);
+}

core/src/main/java/io/kestra/core/runners/Executor.java

@@ -1,197 +1,7 @@
 package io.kestra.core.runners;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import io.kestra.core.models.executions.*;
-import io.kestra.core.models.flows.FlowWithException;
-import io.kestra.core.models.flows.FlowWithSource;
-import io.kestra.core.models.flows.State;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
+import io.kestra.core.server.Service;
 
-import java.util.ArrayList;
-import java.util.List;
+public interface Executor extends Service, Runnable {
 
-// TODO for 2.0: this class is used as a queue consumer (which should have been the ExecutorInterface instead),
-//  a queue message (only in Kafka) and an execution context.
-//  At some point, we should rename it to ExecutorContext and move it to the executor module,
-//  then rename the ExecutorInterface to just Executor (to be used as a queue consumer)
-@Getter
-@AllArgsConstructor
-public class Executor {
-    private Execution execution;
-    private Exception exception;
-    private final List<String> from = new ArrayList<>();
-    private Long offset;
-    @JsonIgnore
-    private boolean executionUpdated = false;
-    private FlowWithSource flow;
-    private final List<TaskRun> nexts = new ArrayList<>();
-    private final List<WorkerTask> workerTasks = new ArrayList<>();
-    private final List<ExecutionDelay> executionDelays = new ArrayList<>();
-    private WorkerTaskResult joinedWorkerTaskResult;
-    private final List<SubflowExecution<?>> subflowExecutions = new ArrayList<>();
-    private final List<SubflowExecutionResult> subflowExecutionResults = new ArrayList<>();
-    private SubflowExecutionResult joinedSubflowExecutionResult;
-    private ExecutionRunning executionRunning;
-    private ExecutionResumed executionResumed;
-    private ExecutionResumed joinedExecutionResumed;
-    private final List<WorkerTrigger> workerTriggers = new ArrayList<>();
-    private WorkerJob workerJobToResubmit;
-    private State.Type originalState;
-    private SubflowExecutionEnd subflowExecutionEnd;
-    private SubflowExecutionEnd joinedSubflowExecutionEnd;
-
-    /**
-     * The sequence id should be incremented each time the execution is persisted after mutation.
-     */
-    private long seqId = 0L;
-
-    /**
-     * List of {@link ExecutionKilled} to be propagated part of the execution.
-     */
-    private List<ExecutionKilledExecution> executionKilled;
-
-    public Executor(Execution execution, Long offset) {
-        this.execution = execution;
-        this.offset = offset;
-        this.originalState = execution.getState().getCurrent();
-    }
-
-    public Executor(Execution execution, Long offset, long seqId) {
-        this.execution = execution;
-        this.offset = offset;
-        this.seqId = seqId;
-        this.originalState = execution.getState().getCurrent();
-    }
-
-    public Executor(WorkerTaskResult workerTaskResult) {
-        this.joinedWorkerTaskResult = workerTaskResult;
-    }
-
-    public Executor(SubflowExecutionResult subflowExecutionResult) {
-        this.joinedSubflowExecutionResult = subflowExecutionResult;
-    }
-
-    public Executor(SubflowExecutionEnd subflowExecutionEnd) {
-        this.joinedSubflowExecutionEnd = subflowExecutionEnd;
-    }
-
-    public Executor(WorkerJob workerJob) {
-        this.workerJobToResubmit = workerJob;
-    }
-
-    public Executor(ExecutionResumed executionResumed) {
-        this.joinedExecutionResumed = executionResumed;
-    }
-
-    public Executor(List<ExecutionKilledExecution> executionKilled) {
-        this.executionKilled = executionKilled;
-    }
-
-    public Boolean canBeProcessed() {
-        return !(this.getException() != null || this.getFlow() == null || this.getFlow() instanceof FlowWithException || this.getFlow().getTasks() == null ||
-            this.getExecution().isDeleted() || this.getExecution().getState().isPaused() || this.getExecution().getState().isBreakpoint() || this.getExecution().getState().isQueued());
-    }
-
-    public Executor withFlow(FlowWithSource flow) {
-        this.flow = flow;
-
-        return this;
-    }
-
-    public Executor withExecution(Execution execution, String from) {
-        this.execution = execution;
-        this.from.add(from);
-        this.executionUpdated = true;
-
-        return this;
-    }
-
-    public Executor withException(Exception exception, String from) {
-        this.exception = exception;
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withTaskRun(List<TaskRun> taskRuns, String from) {
-        this.nexts.addAll(taskRuns);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withWorkerTasks(List<WorkerTask> workerTasks, String from) {
-        this.workerTasks.addAll(workerTasks);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withWorkerTriggers(List<WorkerTrigger> workerTriggers, String from) {
-        this.workerTriggers.addAll(workerTriggers);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withWorkerTaskDelays(List<ExecutionDelay> executionDelays, String from) {
-        this.executionDelays.addAll(executionDelays);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withSubflowExecutions(List<SubflowExecution<?>> subflowExecutions, String from) {
-        this.subflowExecutions.addAll(subflowExecutions);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withSubflowExecutionResults(List<SubflowExecutionResult> subflowExecutionResults, String from) {
-        this.subflowExecutionResults.addAll(subflowExecutionResults);
-        this.from.add(from);
-
-        return this;
-    }
-
-    public Executor withExecutionRunning(ExecutionRunning executionRunning) {
-        this.executionRunning = executionRunning;
-
-        return this;
-    }
-
-    public Executor withExecutionResumed(ExecutionResumed executionResumed) {
-        this.executionResumed = executionResumed;
-        return this;
-    }
-
-    public Executor withExecutionKilled(final List<ExecutionKilledExecution> executionKilled) {
-        this.executionKilled = executionKilled;
-        return this;
-    }
-
-    public Executor withSubflowExecutionEnd(SubflowExecutionEnd subflowExecutionEnd) {
-        this.subflowExecutionEnd = subflowExecutionEnd;
-        return this;
-    }
-
-    public Executor serialize() {
-        return new Executor(
-            this.execution,
-            this.offset,
-            this.seqId
-        );
-    }
-
-    /**
-     * Increments and returns the execution sequence id.
-     *
-     * @return the sequence id.
-     */
-    public long incrementAndGetSeqId() {
-        this.seqId++;
-        return seqId;
-    }
 }

core/src/main/java/io/kestra/core/runners/ExecutorInterface.java

@@ -1,7 +0,0 @@
-package io.kestra.core.runners;
-
-import io.kestra.core.server.Service;
-
-public interface ExecutorInterface extends Service, Runnable {
-
-}

core/src/main/java/io/kestra/core/runners/ExecutorState.java

@@ -1,21 +0,0 @@
-package io.kestra.core.runners;
-
-import io.kestra.core.models.flows.State;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-@Data
-@NoArgsConstructor
-public class ExecutorState {
-    private String executionId;
-    private Map<String, State.Type> workerTaskDeduplication = new ConcurrentHashMap<>();
-    private Map<String, String> childDeduplication = new ConcurrentHashMap<>();
-    private Map<String, State.Type> subflowExecutionDeduplication = new ConcurrentHashMap<>();
-
-    public ExecutorState(String executionId) {
-        this.executionId = executionId;
-    }
-}

core/src/main/java/io/kestra/core/runners/FilesService.java

@@ -82,7 +82,8 @@ public abstract class FilesService {
     }
 
     private static String resolveUniqueNameForFile(final Path path) {
-        String filename = path.getFileName().toString().replace(' ', '+');
-        return IdUtils.from(path.toString()) + "-" + filename;
+        String filename = path.getFileName().toString();
+        String encodedFilename = java.net.URLEncoder.encode(filename, java.nio.charset.StandardCharsets.UTF_8);
+        return IdUtils.from(path.toString()) + "-" + encodedFilename;
     }
 }