mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-25 11:12:12 -05:00
Compare commits
150 Commits
fix/remove
...
v0.23.10
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d318281342 | ||
|
|
3f68749276 | ||
|
|
bc07dfbf1c | ||
|
|
3b3aa495b0 | ||
|
|
a945780e4d | ||
|
|
d512f86927 | ||
|
|
7f355dd730 | ||
|
|
ffa33b1a7a | ||
|
|
a5b4ec3b2e | ||
|
|
5585e9df47 | ||
|
|
f8cb335a16 | ||
|
|
af9129f900 | ||
|
|
177ba35e15 | ||
|
|
b99946deff | ||
|
|
19428ad344 | ||
|
|
162764ff0d | ||
|
|
ccd7b43b97 | ||
|
|
53f881ed60 | ||
|
|
0759aaeae8 | ||
|
|
fc8b389d09 | ||
|
|
8355eb191e | ||
|
|
50f72f8ea3 | ||
|
|
ae14d980a4 | ||
|
|
bc1a08b418 | ||
|
|
e264c0b75d | ||
|
|
fccbb6b648 | ||
|
|
a243c563d3 | ||
|
|
45ad1f6ee4 | ||
|
|
8359bfc680 | ||
|
|
30a808188c | ||
|
|
5121ceb63a | ||
|
|
1dae994910 | ||
|
|
26a82fce95 | ||
|
|
a8584a8a33 | ||
|
|
5737216b34 | ||
|
|
747c424f1f | ||
|
|
33bfc979c5 | ||
|
|
58ceb66cfb | ||
|
|
a08266593f | ||
|
|
d5d5f457b4 | ||
|
|
cacac2239d | ||
|
|
5c45bd5eb5 | ||
|
|
fdf126202c | ||
|
|
0f3c745bb9 | ||
|
|
5a6a0ff3e3 | ||
|
|
f5f88e18ce | ||
|
|
12f521860e | ||
|
|
b6cf3e1f93 | ||
|
|
7125885ea9 | ||
|
|
0b29a4a247 | ||
|
|
0377f87c66 | ||
|
|
06bd0c6380 | ||
|
|
cd39995f24 | ||
|
|
938e156bd5 | ||
|
|
1fb7943738 | ||
|
|
09d648cf86 | ||
|
|
02a22faed4 | ||
|
|
169d6610f5 | ||
|
|
e253958cf4 | ||
|
|
c75f06a036 | ||
|
|
b3b1b7a5cb | ||
|
|
34e07b9e2b | ||
|
|
85b449c926 | ||
|
|
0017ead9b3 | ||
|
|
b0292f02f7 | ||
|
|
202dc7308d | ||
|
|
3273a9a40c | ||
|
|
bd303f4529 | ||
|
|
db57326f0f | ||
|
|
90a576490f | ||
|
|
2cdd968100 | ||
|
|
adfc3bf526 | ||
|
|
3a61f9b1ba | ||
|
|
64e3014426 | ||
|
|
1f68e5f4ed | ||
|
|
9bfa888e36 | ||
|
|
691a77538a | ||
|
|
b07086f553 | ||
|
|
ee12c884e9 | ||
|
|
712d6da84f | ||
|
|
fcc5fa2056 | ||
|
|
dace30ded7 | ||
|
|
2b578f0f94 | ||
|
|
91f958b26b | ||
|
|
d7fc6894fe | ||
|
|
c286348d27 | ||
|
|
de4ec49721 | ||
|
|
1966ac6012 | ||
|
|
a293a37ec9 | ||
|
|
f295724bb6 | ||
|
|
06505ad977 | ||
|
|
cb31ef642f | ||
|
|
c320323371 | ||
|
|
a190cdd0e7 | ||
|
|
0678f7c5e9 | ||
|
|
f39ba5c95e | ||
|
|
b4e334c5d8 | ||
|
|
561380c942 | ||
|
|
68b4867b5a | ||
|
|
cb7f99d107 | ||
|
|
efac7146ff | ||
|
|
11de42c0b8 | ||
|
|
b58d9e10dd | ||
|
|
e25e70d37e | ||
|
|
f2dac28997 | ||
|
|
0ac8819d95 | ||
|
|
d261de0df3 | ||
|
|
02cac65614 | ||
|
|
5064687b7e | ||
|
|
7c8419b266 | ||
|
|
84e4c62c6d | ||
|
|
9aa605e23b | ||
|
|
faa77aed79 | ||
|
|
fdce552528 | ||
|
|
a028a61792 | ||
|
|
023a77a320 | ||
|
|
bfee04bca2 | ||
|
|
3756f01bdf | ||
|
|
c1240d7391 | ||
|
|
ac37ae6032 | ||
|
|
9e51b100b0 | ||
|
|
bc81e01608 | ||
|
|
9f2162c942 | ||
|
|
97992d99ee | ||
|
|
f90f6b8429 | ||
|
|
0f7360ae81 | ||
|
|
938590f31f | ||
|
|
b2d1c84a86 | ||
|
|
d7ca302830 | ||
|
|
8656e852cc | ||
|
|
cc72336350 | ||
|
|
316d89764e | ||
|
|
4873bf4d36 | ||
|
|
204bf7f5e1 | ||
|
|
1e0950fdf8 | ||
|
|
4cddc704f4 | ||
|
|
f2f0e29f93 | ||
|
|
95011e022e | ||
|
|
65503b708a | ||
|
|
876b8cb2e6 | ||
|
|
f3b7592dfa | ||
|
|
4dbeaf86bb | ||
|
|
f98e78399d | ||
|
|
71dac0f311 | ||
|
|
3077d0ac7a | ||
|
|
9504bbaffe | ||
|
|
159c9373ad | ||
|
|
55b9088b55 | ||
|
|
601d1a0abb | ||
|
|
4a1cf98f26 |
3
.github/workflows/main.yml
vendored
3
.github/workflows/main.yml
vendored
@@ -43,6 +43,9 @@ jobs:
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
6
.github/workflows/setversion-tag.yml
vendored
6
.github/workflows/setversion-tag.yml
vendored
@@ -22,11 +22,11 @@ jobs:
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
RELEASE_BRANCH="refs/heads/releases/v${BASE_VERSION}.x"
|
||||
|
||||
|
||||
CURRENT_BRANCH="$GITHUB_REF"
|
||||
if ! [[ "$CURRENT_BRANCH" == "$RELEASE_BRANCH" ]]; then
|
||||
echo "Invalid release branch. Expected $RELEASE_BRANCH, was $CURRENT_BRANCH"
|
||||
@@ -54,4 +54,4 @@ jobs:
|
||||
git commit -m"chore(version): update to version '$RELEASE_VERSION'"
|
||||
git push
|
||||
git tag -a "v$RELEASE_VERSION" -m"v$RELEASE_VERSION"
|
||||
git push origin "v$RELEASE_VERSION"
|
||||
git push --tags
|
||||
38
.github/workflows/workflow-github-release.yml
vendored
38
.github/workflows/workflow-github-release.yml
vendored
@@ -6,23 +6,15 @@ on:
|
||||
GH_PERSONAL_TOKEN:
|
||||
description: "The Github personal token."
|
||||
required: true
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
SLACK_RELEASES_WEBHOOK_URL:
|
||||
description: "The Slack webhook URL."
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Github - Release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Download Exec
|
||||
- name: Artifacts - Download executable
|
||||
uses: actions/download-artifact@v4
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable
|
||||
|
||||
# Check out
|
||||
- name: Checkout - Repository
|
||||
uses: actions/checkout@v4
|
||||
@@ -36,11 +28,20 @@ jobs:
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
sparse-checkout-cone-mode: true
|
||||
ref: fix/core-release
|
||||
path: actions
|
||||
sparse-checkout: |
|
||||
.github/actions
|
||||
|
||||
# Download Exec
|
||||
# Must be done after checkout actions
|
||||
- name: Artifacts - Download executable
|
||||
uses: actions/download-artifact@v4
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable
|
||||
|
||||
|
||||
# GitHub Release
|
||||
- name: Create GitHub release
|
||||
uses: ./actions/.github/actions/github-release
|
||||
@@ -49,3 +50,16 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
|
||||
# Trigger gha workflow to bump helm chart version
|
||||
- name: GitHub - Trigger the Helm chart version bump
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
with:
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
repository: kestra-io/helm-charts
|
||||
event-type: update-helm-chart-version
|
||||
client-payload: |-
|
||||
{
|
||||
"new_version": "${{ github.ref_name }}",
|
||||
"github_repository": "${{ github.repository }}",
|
||||
"github_actor": "${{ github.actor }}"
|
||||
}
|
||||
6
.github/workflows/workflow-publish-maven.yml
vendored
6
.github/workflows/workflow-publish-maven.yml
vendored
@@ -39,8 +39,8 @@ jobs:
|
||||
- name: Publish - Release package to Maven Central
|
||||
shell: bash
|
||||
env:
|
||||
ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
|
||||
ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SONATYPE_USER }}
|
||||
ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE}}
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
|
||||
echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
|
||||
echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
|
||||
./gradlew publishToSonatype ${{ startsWith(github.ref, 'refs/tags/v') && 'closeAndReleaseSonatypeStagingRepository' || '' }}
|
||||
./gradlew publishToMavenCentral
|
||||
|
||||
# Gradle dependency
|
||||
- name: Java - Gradle dependency graph
|
||||
|
||||
9
.github/workflows/workflow-release.yml
vendored
9
.github/workflows/workflow-release.yml
vendored
@@ -42,6 +42,12 @@ on:
|
||||
SONATYPE_GPG_FILE:
|
||||
description: "The Sonatype GPG file."
|
||||
required: true
|
||||
GH_PERSONAL_TOKEN:
|
||||
description: "The Github personal token."
|
||||
required: true
|
||||
SLACK_RELEASES_WEBHOOK_URL:
|
||||
description: "The Slack webhook URL."
|
||||
required: true
|
||||
jobs:
|
||||
build-artifacts:
|
||||
name: Build - Artifacts
|
||||
@@ -77,4 +83,5 @@ jobs:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: ./.github/workflows/workflow-github-release.yml
|
||||
secrets:
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
1
.plugins
1
.plugins
@@ -61,6 +61,7 @@
|
||||
#plugin-jenkins:io.kestra.plugin:plugin-jenkins:LATEST
|
||||
#plugin-jira:io.kestra.plugin:plugin-jira:LATEST
|
||||
#plugin-kafka:io.kestra.plugin:plugin-kafka:LATEST
|
||||
#plugin-kestra:io.kestra.plugin:plugin-kestra:LATEST
|
||||
#plugin-kubernetes:io.kestra.plugin:plugin-kubernetes:LATEST
|
||||
#plugin-langchain4j:io.kestra.plugin:plugin-langchain4j:LATEST
|
||||
#plugin-ldap:io.kestra.plugin:plugin-ldap:LATEST
|
||||
|
||||
141
build.gradle
141
build.gradle
@@ -31,12 +31,10 @@ plugins {
|
||||
id 'com.github.node-gradle.node' version '7.1.0'
|
||||
|
||||
// release
|
||||
id "io.github.gradle-nexus.publish-plugin" version "2.0.0"
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.0"
|
||||
id 'signing'
|
||||
id 'ru.vyarus.pom' version '3.0.0' apply false
|
||||
id 'ru.vyarus.github-info' version '2.0.0' apply false
|
||||
id "com.vanniktech.maven.publish" version "0.33.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.1" apply false
|
||||
@@ -414,6 +412,7 @@ distTar.dependsOn shadowJar
|
||||
startScripts.dependsOn shadowJar
|
||||
startShadowScripts.dependsOn jar
|
||||
shadowJar.dependsOn 'ui:assembleFrontend'
|
||||
shadowJar.dependsOn jar
|
||||
|
||||
/**********************************************************************************************************************\
|
||||
* Executable Jar
|
||||
@@ -484,24 +483,11 @@ tasks.register('runStandalone', JavaExec) {
|
||||
/**********************************************************************************************************************\
|
||||
* Publish
|
||||
**********************************************************************************************************************/
|
||||
nexusPublishing {
|
||||
repositoryDescription = "${project.group}:${rootProject.name}:${project.version}"
|
||||
useStaging = !project.version.endsWith("-SNAPSHOT")
|
||||
repositories {
|
||||
sonatype {
|
||||
nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
|
||||
snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
|
||||
}
|
||||
}
|
||||
}
|
||||
subprojects {subProject ->
|
||||
|
||||
subprojects {
|
||||
|
||||
if (it.name != 'jmh-benchmarks') {
|
||||
apply plugin: "maven-publish"
|
||||
if (subProject.name != 'jmh-benchmarks' && subProject.name != rootProject.name) {
|
||||
apply plugin: 'signing'
|
||||
apply plugin: 'ru.vyarus.pom'
|
||||
apply plugin: 'ru.vyarus.github-info'
|
||||
apply plugin: "com.vanniktech.maven.publish"
|
||||
|
||||
javadoc {
|
||||
options {
|
||||
@@ -535,56 +521,98 @@ subprojects {
|
||||
}
|
||||
}
|
||||
|
||||
github {
|
||||
user 'kestra-io'
|
||||
license 'Apache'
|
||||
repository 'kestra'
|
||||
site 'https://kestra.io'
|
||||
//These modules should not be published
|
||||
def unpublishedModules = ["jdbc-mysql", "jdbc-postgres", "webserver"]
|
||||
if (subProject.name in unpublishedModules){
|
||||
return
|
||||
}
|
||||
|
||||
maven.pom {
|
||||
description = 'The modern, scalable orchestrator & scheduler open source platform'
|
||||
mavenPublishing {
|
||||
publishToMavenCentral(true)
|
||||
signAllPublications()
|
||||
|
||||
developers {
|
||||
developer {
|
||||
id = "tchiotludo"
|
||||
name = "Ludovic Dehon"
|
||||
coordinates(
|
||||
"${rootProject.group}",
|
||||
subProject.name == "cli" ? rootProject.name : subProject.name,
|
||||
"${rootProject.version}"
|
||||
)
|
||||
|
||||
pom {
|
||||
name = project.name
|
||||
description = "${project.group}:${project.name}:${rootProject.version}"
|
||||
url = "https://github.com/kestra-io/${rootProject.name}"
|
||||
|
||||
licenses {
|
||||
license {
|
||||
name = "The Apache License, Version 2.0"
|
||||
url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
|
||||
}
|
||||
}
|
||||
developers {
|
||||
developer {
|
||||
id = "tchiotludo"
|
||||
name = "Ludovic Dehon"
|
||||
email = "ldehon@kestra.io"
|
||||
}
|
||||
}
|
||||
scm {
|
||||
connection = 'scm:git:'
|
||||
url = "https://github.com/kestra-io/${rootProject.name}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
publishing {
|
||||
publications {
|
||||
sonatypePublication(MavenPublication) {
|
||||
version project.version
|
||||
afterEvaluate {
|
||||
publishing {
|
||||
publications {
|
||||
withType(MavenPublication).configureEach { publication ->
|
||||
|
||||
if (project.name.contains('cli')) {
|
||||
groupId "io.kestra"
|
||||
artifactId "kestra"
|
||||
|
||||
artifact shadowJar
|
||||
artifact executableJar
|
||||
} else if (project.name.contains('platform')){
|
||||
groupId project.group
|
||||
artifactId project.name
|
||||
} else {
|
||||
from components.java
|
||||
|
||||
groupId project.group
|
||||
artifactId project.name
|
||||
|
||||
artifact sourcesJar
|
||||
artifact javadocJar
|
||||
artifact testsJar
|
||||
if (subProject.name == "platform") {
|
||||
// Clear all artifacts except the BOM
|
||||
publication.artifacts.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
signing {
|
||||
// only sign JARs that we publish to Sonatype
|
||||
required { gradle.taskGraph.hasTask("publishSonatypePublicationPublicationToSonatypeRepository") }
|
||||
sign publishing.publications.sonatypePublication
|
||||
if (subProject.name == 'cli') {
|
||||
|
||||
/* Make sure the special publication is wired *after* every plugin */
|
||||
subProject.afterEvaluate {
|
||||
/* 1. Remove the default java component so Gradle stops expecting
|
||||
the standard cli-*.jar, sources, javadoc, etc. */
|
||||
components.removeAll { it.name == "java" }
|
||||
|
||||
/* 2. Replace the publication’s artifacts with shadow + exec */
|
||||
publishing.publications.withType(MavenPublication).configureEach { pub ->
|
||||
pub.artifacts.clear()
|
||||
|
||||
// main shadow JAR built at root
|
||||
pub.artifact(rootProject.tasks.named("shadowJar").get()) {
|
||||
extension = "jar"
|
||||
}
|
||||
|
||||
// executable ZIP built at root
|
||||
pub.artifact(rootProject.tasks.named("executableJar").get().archiveFile) {
|
||||
classifier = "exec"
|
||||
extension = "zip"
|
||||
}
|
||||
pub.artifact(tasks.named("sourcesJar").get())
|
||||
pub.artifact(tasks.named("javadocJar").get())
|
||||
|
||||
}
|
||||
|
||||
/* 3. Disable Gradle-module metadata for this publication to
|
||||
avoid the “artifact removed from java component” error. */
|
||||
tasks.withType(GenerateModuleMetadata).configureEach { it.enabled = false }
|
||||
|
||||
/* 4. Make every publish task in :cli wait for the two artifacts */
|
||||
tasks.matching { it.name.startsWith("publish") }.configureEach {
|
||||
dependsOn rootProject.tasks.named("shadowJar")
|
||||
dependsOn rootProject.tasks.named("executableJar")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType(GenerateModuleMetadata).configureEach {
|
||||
@@ -595,6 +623,7 @@ subprojects {
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**********************************************************************************************************************\
|
||||
* Version
|
||||
**********************************************************************************************************************/
|
||||
|
||||
@@ -37,4 +37,4 @@ dependencies {
|
||||
|
||||
//test
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
package io.kestra.cli;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
import io.micronaut.core.annotation.Nullable;
|
||||
import io.micronaut.http.HttpHeaders;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
@@ -16,16 +14,15 @@ import io.micronaut.http.netty.body.NettyJsonHandler;
|
||||
import io.micronaut.json.JsonMapper;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import lombok.Builder;
|
||||
import lombok.Value;
|
||||
import lombok.extern.jackson.Jacksonized;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.Builder;
|
||||
import lombok.Value;
|
||||
import lombok.extern.jackson.Jacksonized;
|
||||
import picocli.CommandLine;
|
||||
|
||||
public abstract class AbstractApiCommand extends AbstractCommand {
|
||||
@CommandLine.Option(names = {"--server"}, description = "Kestra server url", defaultValue = "http://localhost:8080")
|
||||
@@ -37,7 +34,7 @@ public abstract class AbstractApiCommand extends AbstractCommand {
|
||||
@CommandLine.Option(names = {"--user"}, paramLabel = "<user:password>", description = "Server user and password")
|
||||
protected String user;
|
||||
|
||||
@CommandLine.Option(names = {"--tenant"}, description = "Tenant identifier (EE only, when multi-tenancy is enabled)")
|
||||
@CommandLine.Option(names = {"--tenant"}, description = "Tenant identifier (EE only)")
|
||||
protected String tenantId;
|
||||
|
||||
@CommandLine.Option(names = {"--api-token"}, description = "API Token (EE only).")
|
||||
@@ -87,12 +84,12 @@ public abstract class AbstractApiCommand extends AbstractCommand {
|
||||
return request;
|
||||
}
|
||||
|
||||
protected String apiUri(String path) {
|
||||
protected String apiUri(String path, String tenantId) {
|
||||
if (path == null || !path.startsWith("/")) {
|
||||
throw new IllegalArgumentException("'path' must be non-null and start with '/'");
|
||||
}
|
||||
|
||||
return tenantId == null ? "/api/v1/" + MAIN_TENANT + path : "/api/v1/" + tenantId + path;
|
||||
return "/api/v1/" + tenantId + path;
|
||||
}
|
||||
|
||||
@Builder
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package io.kestra.cli;
|
||||
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.models.validations.ModelValidator;
|
||||
import io.kestra.core.models.validations.ValidateConstraintViolation;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
@@ -9,6 +10,7 @@ import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -31,6 +33,9 @@ public abstract class AbstractValidateCommand extends AbstractApiCommand {
|
||||
@CommandLine.Parameters(index = "0", description = "the directory containing files to check")
|
||||
protected Path directory;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
/** {@inheritDoc} **/
|
||||
@Override
|
||||
protected boolean loadExternalPlugins() {
|
||||
@@ -112,7 +117,7 @@ public abstract class AbstractValidateCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows/validate"), body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows/validate", tenantService.getTenantId(tenantId)), body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
List<ValidateConstraintViolation> validations = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -2,11 +2,13 @@ package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@@ -23,6 +25,9 @@ public class FlowCreateCommand extends AbstractApiCommand {
|
||||
@CommandLine.Parameters(index = "0", description = "The file containing the flow")
|
||||
public Path flowFile;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -34,7 +39,7 @@ public class FlowCreateCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows"), body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows", tenantService.getTenantId(tenantId)), body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -2,10 +2,12 @@ package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@@ -23,6 +25,9 @@ public class FlowDeleteCommand extends AbstractApiCommand {
|
||||
@CommandLine.Parameters(index = "1", description = "The ID of the flow")
|
||||
public String id;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -30,7 +35,7 @@ public class FlowDeleteCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.DELETE(apiUri("/flows/" + namespace + "/" + id ));
|
||||
.DELETE(apiUri("/flows/" + namespace + "/" + id, tenantService.getTenantId(tenantId)));
|
||||
|
||||
client.toBlocking().exchange(
|
||||
this.requestOptions(request)
|
||||
|
||||
@@ -2,7 +2,7 @@ package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.HttpResponse;
|
||||
import io.micronaut.http.MediaType;
|
||||
@@ -25,9 +25,8 @@ import java.nio.file.Path;
|
||||
public class FlowExportCommand extends AbstractApiCommand {
|
||||
private static final String DEFAULT_FILE_NAME = "flows.zip";
|
||||
|
||||
// @FIXME: Keep it for bug in micronaut that need to have inject on top level command to inject on abstract classe
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@CommandLine.Option(names = {"--namespace"}, description = "The namespace of flows to export")
|
||||
public String namespace;
|
||||
@@ -41,7 +40,7 @@ public class FlowExportCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<Object> request = HttpRequest
|
||||
.GET(apiUri("/flows/export/by-query") + (namespace != null ? "?namespace=" + namespace : ""))
|
||||
.GET(apiUri("/flows/export/by-query", tenantService.getTenantId(tenantId)) + (namespace != null ? "?namespace=" + namespace : ""))
|
||||
.accept(MediaType.APPLICATION_OCTET_STREAM);
|
||||
|
||||
HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package io.kestra.cli.commands.flows;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.LocalFlowRepositoryLoader;
|
||||
@@ -30,7 +31,7 @@ import java.util.concurrent.TimeoutException;
|
||||
description = "Test a flow"
|
||||
)
|
||||
@Slf4j
|
||||
public class FlowTestCommand extends AbstractCommand {
|
||||
public class FlowTestCommand extends AbstractApiCommand {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
@@ -76,6 +77,7 @@ public class FlowTestCommand extends AbstractCommand {
|
||||
FlowRepositoryInterface flowRepository = applicationContext.getBean(FlowRepositoryInterface.class);
|
||||
FlowInputOutput flowInputOutput = applicationContext.getBean(FlowInputOutput.class);
|
||||
RunnerUtils runnerUtils = applicationContext.getBean(RunnerUtils.class);
|
||||
TenantIdSelectorService tenantService = applicationContext.getBean(TenantIdSelectorService.class);
|
||||
|
||||
Map<String, Object> inputs = new HashMap<>();
|
||||
|
||||
@@ -89,7 +91,7 @@ public class FlowTestCommand extends AbstractCommand {
|
||||
|
||||
try {
|
||||
runner.run();
|
||||
repositoryLoader.load(file.toFile());
|
||||
repositoryLoader.load(tenantService.getTenantId(tenantId), file.toFile());
|
||||
|
||||
List<Flow> all = flowRepository.findAllForAllTenants();
|
||||
if (all.size() != 1) {
|
||||
|
||||
@@ -2,11 +2,13 @@ package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@@ -29,6 +31,9 @@ public class FlowUpdateCommand extends AbstractApiCommand {
|
||||
@CommandLine.Parameters(index = "2", description = "The ID of the flow")
|
||||
public String id;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -40,7 +45,7 @@ public class FlowUpdateCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.PUT(apiUri("/flows/" + namespace + "/" + id ), body).contentType(MediaType.APPLICATION_YAML);
|
||||
.PUT(apiUri("/flows/" + namespace + "/" + id, tenantService.getTenantId(tenantId)), body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -2,6 +2,7 @@ package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
import io.micronaut.core.type.Argument;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
@@ -9,6 +10,7 @@ import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
@@ -36,6 +38,9 @@ public class FlowUpdatesCommand extends AbstractApiCommand {
|
||||
@CommandLine.Option(names = {"--namespace"}, description = "The parent namespace of the flows, if not set, every namespace are allowed.")
|
||||
public String namespace;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantIdSelectorService;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -66,7 +71,7 @@ public class FlowUpdatesCommand extends AbstractApiCommand {
|
||||
namespaceQuery = "&namespace=" + namespace;
|
||||
}
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows/bulk") + "?allowNamespaceChild=true&delete=" + delete + namespaceQuery, body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows/bulk", tenantIdSelectorService.getTenantId(tenantId)) + "?allowNamespaceChild=true&delete=" + delete + namespaceQuery, body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
List<UpdateResult> updated = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.kestra.cli.commands.flows;
|
||||
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.validations.ModelValidator;
|
||||
import io.kestra.core.services.FlowService;
|
||||
@@ -22,6 +23,9 @@ public class FlowValidateCommand extends AbstractValidateCommand {
|
||||
@Inject
|
||||
private FlowService flowService;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
return this.call(
|
||||
@@ -35,7 +39,7 @@ public class FlowValidateCommand extends AbstractValidateCommand {
|
||||
FlowWithSource flow = (FlowWithSource) object;
|
||||
List<String> warnings = new ArrayList<>();
|
||||
warnings.addAll(flowService.deprecationPaths(flow).stream().map(deprecation -> deprecation + " is deprecated").toList());
|
||||
warnings.addAll(flowService.warnings(flow, this.tenantId));
|
||||
warnings.addAll(flowService.warnings(flow, tenantService.getTenantId(tenantId)));
|
||||
return warnings;
|
||||
},
|
||||
(Object object) -> {
|
||||
|
||||
@@ -3,6 +3,7 @@ package io.kestra.cli.commands.flows.namespaces;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
|
||||
import io.kestra.cli.commands.flows.IncludeHelperExpander;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
import io.micronaut.core.type.Argument;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
@@ -10,6 +11,7 @@ import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
@@ -30,6 +32,9 @@ public class FlowNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCo
|
||||
@CommandLine.Option(names = {"--override-namespaces"}, negatable = true, description = "Replace namespace of all flows by the one provided")
|
||||
public boolean override = false;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -59,7 +64,7 @@ public class FlowNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCo
|
||||
}
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows/") + namespace + "?delete=" + delete, body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows/", tenantService.getTenantId(tenantId)) + namespace + "?delete=" + delete, body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
List<UpdateResult> updated = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -2,12 +2,14 @@ package io.kestra.cli.commands.namespaces.files;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.utils.KestraIgnore;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.multipart.MultipartBody;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@@ -34,6 +36,9 @@ public class NamespaceFilesUpdateCommand extends AbstractApiCommand {
|
||||
@CommandLine.Option(names = {"--delete"}, negatable = true, description = "Whether missing should be deleted")
|
||||
public boolean delete = false;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
private static final String KESTRA_IGNORE_FILE = ".kestraignore";
|
||||
|
||||
@Override
|
||||
@@ -44,7 +49,7 @@ public class NamespaceFilesUpdateCommand extends AbstractApiCommand {
|
||||
|
||||
try (var files = Files.walk(from); DefaultHttpClient client = client()) {
|
||||
if (delete) {
|
||||
client.toBlocking().exchange(this.requestOptions(HttpRequest.DELETE(apiUri("/namespaces/") + namespace + "/files?path=" + to, null)));
|
||||
client.toBlocking().exchange(this.requestOptions(HttpRequest.DELETE(apiUri("/namespaces/", tenantService.getTenantId(tenantId)) + namespace + "/files?path=" + to, null)));
|
||||
}
|
||||
|
||||
KestraIgnore kestraIgnore = new KestraIgnore(from);
|
||||
@@ -62,7 +67,7 @@ public class NamespaceFilesUpdateCommand extends AbstractApiCommand {
|
||||
client.toBlocking().exchange(
|
||||
this.requestOptions(
|
||||
HttpRequest.POST(
|
||||
apiUri("/namespaces/") + namespace + "/files?path=" + destination,
|
||||
apiUri("/namespaces/", tenantService.getTenantId(tenantId)) + namespace + "/files?path=" + destination,
|
||||
body
|
||||
).contentType(MediaType.MULTIPART_FORM_DATA)
|
||||
)
|
||||
|
||||
@@ -3,11 +3,13 @@ package io.kestra.cli.commands.namespaces.kv;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MediaType;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Option;
|
||||
@@ -42,6 +44,9 @@ public class KvUpdateCommand extends AbstractApiCommand {
|
||||
@Option(names = {"-f", "--file-value"}, description = "The file from which to read the value to set. If this is provided, it will take precedence over any specified value.")
|
||||
public Path fileValue;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
super.call();
|
||||
@@ -56,7 +61,7 @@ public class KvUpdateCommand extends AbstractApiCommand {
|
||||
|
||||
Duration ttl = expiration == null ? null : Duration.parse(expiration);
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.PUT(apiUri("/namespaces/") + namespace + "/kv/" + key, value)
|
||||
.PUT(apiUri("/namespaces/", tenantService.getTenantId(tenantId)) + namespace + "/kv/" + key, value)
|
||||
.contentType(MediaType.APPLICATION_JSON_TYPE);
|
||||
|
||||
if (ttl != null) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.cli.services.FileChangedEventListener;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import io.kestra.core.models.ServerType;
|
||||
import io.kestra.core.repositories.LocalFlowRepositoryLoader;
|
||||
@@ -44,6 +45,9 @@ public class StandAloneCommand extends AbstractServerCommand {
|
||||
@CommandLine.Option(names = {"-f", "--flow-path"}, description = "the flow path containing flow to inject at startup (when running with a memory flow repository)")
|
||||
private File flowPath;
|
||||
|
||||
@CommandLine.Option(names = "--tenant", description = "Tenant identifier, Required to load flows from path with the enterprise edition")
|
||||
private String tenantId;
|
||||
|
||||
@CommandLine.Option(names = {"--worker-thread"}, description = "the number of worker threads, defaults to four times the number of available processors. Set it to 0 to avoid starting a worker.")
|
||||
private int workerThread = defaultWorkerThread();
|
||||
|
||||
@@ -98,7 +102,8 @@ public class StandAloneCommand extends AbstractServerCommand {
|
||||
if (flowPath != null) {
|
||||
try {
|
||||
LocalFlowRepositoryLoader localFlowRepositoryLoader = applicationContext.getBean(LocalFlowRepositoryLoader.class);
|
||||
localFlowRepositoryLoader.load(null, this.flowPath);
|
||||
TenantIdSelectorService tenantIdSelectorService = applicationContext.getBean(TenantIdSelectorService.class);
|
||||
localFlowRepositoryLoader.load(tenantIdSelectorService.getTenantId(this.tenantId), this.flowPath);
|
||||
} catch (IOException e) {
|
||||
throw new CommandLine.ParameterException(this.spec.commandLine(), "Invalid flow path", e);
|
||||
}
|
||||
|
||||
@@ -2,8 +2,8 @@ package io.kestra.cli.commands.templates;
|
||||
|
||||
import io.kestra.cli.AbstractApiCommand;
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.models.templates.TemplateEnabled;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.HttpResponse;
|
||||
import io.micronaut.http.MediaType;
|
||||
@@ -27,9 +27,8 @@ import java.nio.file.Path;
|
||||
public class TemplateExportCommand extends AbstractApiCommand {
|
||||
private static final String DEFAULT_FILE_NAME = "templates.zip";
|
||||
|
||||
// @FIXME: Keep it for bug in micronaut that need to have inject on top level command to inject on abstract classe
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@CommandLine.Option(names = {"--namespace"}, description = "The namespace of templates to export")
|
||||
public String namespace;
|
||||
@@ -43,7 +42,7 @@ public class TemplateExportCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<Object> request = HttpRequest
|
||||
.GET(apiUri("/templates/export/by-query") + (namespace != null ? "?namespace=" + namespace : ""))
|
||||
.GET(apiUri("/templates/export/by-query", tenantService.getTenantId(tenantId)) + (namespace != null ? "?namespace=" + namespace : ""))
|
||||
.accept(MediaType.APPLICATION_OCTET_STREAM);
|
||||
|
||||
HttpResponse<byte[]> response = client.toBlocking().exchange(this.requestOptions(request), byte[].class);
|
||||
|
||||
@@ -2,6 +2,7 @@ package io.kestra.cli.commands.templates.namespaces;
|
||||
|
||||
import io.kestra.cli.AbstractValidateCommand;
|
||||
import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
|
||||
import io.kestra.cli.services.TenantIdSelectorService;
|
||||
import io.kestra.core.models.templates.Template;
|
||||
import io.kestra.core.models.templates.TemplateEnabled;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
@@ -10,6 +11,7 @@ import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.MutableHttpRequest;
|
||||
import io.micronaut.http.client.exceptions.HttpClientResponseException;
|
||||
import io.micronaut.http.client.netty.DefaultHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@@ -27,6 +29,9 @@ import jakarta.validation.ConstraintViolationException;
|
||||
@TemplateEnabled
|
||||
public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
super.call();
|
||||
@@ -44,7 +49,7 @@ public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpda
|
||||
|
||||
try (DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<List<Template>> request = HttpRequest
|
||||
.POST(apiUri("/templates/") + namespace + "?delete=" + delete, templates);
|
||||
.POST(apiUri("/templates/", tenantService.getTenantId(tenantId)) + namespace + "?delete=" + delete, templates);
|
||||
|
||||
List<UpdateResult> updated = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -12,8 +12,8 @@ import io.kestra.core.services.PluginDefaultService;
|
||||
import io.micronaut.context.annotation.Requires;
|
||||
import io.micronaut.context.annotation.Value;
|
||||
import io.micronaut.scheduling.io.watch.FileWatchConfiguration;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -26,6 +26,8 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
@Requires(property = "micronaut.io.watch.enabled", value = "true")
|
||||
@@ -111,6 +113,8 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
public void startListening(List<Path> paths) throws IOException, InterruptedException {
|
||||
String tenantId = this.tenantId != null ? this.tenantId : MAIN_TENANT;
|
||||
|
||||
for (Path path : paths) {
|
||||
path.register(watchService, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_DELETE, StandardWatchEventKinds.ENTRY_MODIFY);
|
||||
}
|
||||
@@ -189,6 +193,8 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private void loadFlowsFromFolder(Path folder) {
|
||||
String tenantId = this.tenantId != null ? this.tenantId : MAIN_TENANT;
|
||||
|
||||
try {
|
||||
Files.walkFileTree(folder, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
@@ -232,6 +238,8 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private Optional<FlowWithSource> parseFlow(String content, Path entry) {
|
||||
String tenantId = this.tenantId != null ? this.tenantId : MAIN_TENANT;
|
||||
|
||||
try {
|
||||
FlowWithSource flow = pluginDefaultService.parseFlowWithAllDefaults(tenantId, content, false);
|
||||
modelValidator.validate(flow);
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package io.kestra.cli.services;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
import io.kestra.core.exceptions.KestraRuntimeException;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
@Singleton
|
||||
public class TenantIdSelectorService {
|
||||
|
||||
//For override purpose in Kestra EE
|
||||
public String getTenantId(String tenantId) {
|
||||
if (StringUtils.isNotBlank(tenantId) && !MAIN_TENANT.equals(tenantId)){
|
||||
throw new KestraRuntimeException("Tenant id can only be 'main'");
|
||||
}
|
||||
return MAIN_TENANT;
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,9 @@ micronaut:
|
||||
static:
|
||||
paths: classpath:static
|
||||
mapping: /static/**
|
||||
root:
|
||||
paths: classpath:root
|
||||
mapping: /**
|
||||
server:
|
||||
max-request-size: 10GB
|
||||
multipart:
|
||||
|
||||
@@ -108,6 +108,34 @@ class FlowCreateOrUpdateCommandTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_fail_with_incorrect_tenant() {
|
||||
URL directory = FlowCreateOrUpdateCommandTest.class.getClassLoader().getResource("flows");
|
||||
|
||||
ByteArrayOutputStream err = new ByteArrayOutputStream();
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
|
||||
|
||||
EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
|
||||
embeddedServer.start();
|
||||
|
||||
String[] args = {
|
||||
"--server",
|
||||
embeddedServer.getURL().toString(),
|
||||
"--user",
|
||||
"myuser:pass:word",
|
||||
"--tenant", "incorrect",
|
||||
directory.getPath(),
|
||||
|
||||
};
|
||||
PicocliRunner.call(FlowUpdatesCommand.class, ctx, args);
|
||||
|
||||
assertThat(err.toString()).contains("Tenant id can only be 'main'");
|
||||
err.reset();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void helper() {
|
||||
URL directory = FlowCreateOrUpdateCommandTest.class.getClassLoader().getResource("helper");
|
||||
|
||||
@@ -21,7 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class PluginDocCommandTest {
|
||||
|
||||
public static final String PLUGIN_TEMPLATE_TEST = "plugin-template-test-0.18.0-SNAPSHOT.jar";
|
||||
public static final String PLUGIN_TEMPLATE_TEST = "plugin-template-test-0.24.0-SNAPSHOT.jar";
|
||||
|
||||
@Test
|
||||
void run() throws IOException, URISyntaxException {
|
||||
|
||||
@@ -20,7 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class PluginListCommandTest {
|
||||
|
||||
private static final String PLUGIN_TEMPLATE_TEST = "plugin-template-test-0.18.0-SNAPSHOT.jar";
|
||||
private static final String PLUGIN_TEMPLATE_TEST = "plugin-template-test-0.24.0-SNAPSHOT.jar";
|
||||
|
||||
@Test
|
||||
void shouldListPluginsInstalledLocally() throws IOException, URISyntaxException {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package io.kestra.cli.commands.servers;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import io.kestra.cli.App;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.context.env.Environment;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TenantIdSelectorServiceTest {
|
||||
|
||||
@Test
|
||||
void should_fail_without_tenant_id() {
|
||||
ByteArrayOutputStream err = new ByteArrayOutputStream();
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
|
||||
String[] start = {
|
||||
"server", "standalone",
|
||||
"-f", "unused",
|
||||
"--tenant", "wrong_tenant"
|
||||
};
|
||||
PicocliRunner.call(App.class, ctx, start);
|
||||
|
||||
assertThat(err.toString()).contains("Tenant id can only be 'main'");
|
||||
err.reset();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -284,7 +284,7 @@ public class HttpClient implements Closeable {
|
||||
} else if (cls.isAssignableFrom(Byte[].class)) {
|
||||
return (T) ArrayUtils.toObject(EntityUtils.toByteArray(entity));
|
||||
} else {
|
||||
return (T) JacksonMapper.ofJson().readValue(entity.getContent(), cls);
|
||||
return (T) JacksonMapper.ofJson(false).readValue(entity.getContent(), cls);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package io.kestra.core.metrics;
|
||||
|
||||
import io.kestra.core.models.ServerType;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import io.micronaut.configuration.metrics.aggregator.MeterRegistryConfigurer;
|
||||
import io.micronaut.context.annotation.Requires;
|
||||
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import io.micronaut.context.annotation.Value;
|
||||
import io.micronaut.core.annotation.Nullable;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
@@ -15,20 +18,26 @@ public class GlobalTagsConfigurer implements MeterRegistryConfigurer<SimpleMeter
|
||||
@Inject
|
||||
MetricConfig metricConfig;
|
||||
|
||||
@Nullable
|
||||
@Value("${kestra.server-type}")
|
||||
ServerType serverType;
|
||||
|
||||
@Override
|
||||
public void configure(SimpleMeterRegistry meterRegistry) {
|
||||
if (metricConfig.getTags() != null) {
|
||||
meterRegistry
|
||||
.config()
|
||||
.commonTags(
|
||||
metricConfig.getTags()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
|
||||
.toList()
|
||||
.toArray(String[]::new)
|
||||
);
|
||||
}
|
||||
String[] tags = Stream
|
||||
.concat(
|
||||
metricConfig.getTags() != null ? metricConfig.getTags()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.flatMap(e -> Stream.of(e.getKey(), e.getValue())) : Stream.empty(),
|
||||
serverType != null ? Stream.of("server_type", serverType.name()) : Stream.empty()
|
||||
)
|
||||
.toList()
|
||||
.toArray(String[]::new);
|
||||
|
||||
meterRegistry
|
||||
.config()
|
||||
.commonTags(tags);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public record Label(@NotNull String key, @NotNull String value) {
|
||||
@@ -29,11 +28,36 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
* @return the nested {@link Map}.
|
||||
*/
|
||||
public static Map<String, Object> toNestedMap(List<Label> labels) {
|
||||
Map<String, Object> asMap = labels.stream()
|
||||
return MapUtils.flattenToNestedMap(toMap(labels));
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper method for converting a list of labels to a flat map.
|
||||
* Key order is kept.
|
||||
*
|
||||
* @param labels The list of {@link Label} to be converted.
|
||||
* @return the flat {@link Map}.
|
||||
*/
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
// using an accumulator in case labels with the same key exists: the first is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> first));
|
||||
return MapUtils.flattenToNestedMap(asMap);
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper method for deduplicating a list of labels by their key.
|
||||
* Value of the last key occurrence is kept.
|
||||
*
|
||||
* @param labels The list of {@link Label} to be deduplicated.
|
||||
* @return the deduplicated {@link List}.
|
||||
*/
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -125,7 +125,7 @@ public record QueryFilter(
|
||||
END_DATE("endDate") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.GREATER_THAN, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS);
|
||||
return List.of(Op.GREATER_THAN_OR_EQUAL_TO, Op.GREATER_THAN, Op.LESS_THAN_OR_EQUAL_TO, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
STATE("state") {
|
||||
|
||||
@@ -15,6 +15,8 @@ import jakarta.validation.constraints.NotNull;
|
||||
@NoArgsConstructor
|
||||
public class Setting {
|
||||
public static final String INSTANCE_UUID = "instance.uuid";
|
||||
public static final String INSTANCE_VERSION = "instance.version";
|
||||
|
||||
@NotNull
|
||||
private String key;
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import io.kestra.core.serializers.ListOrMapOfLabelSerializer;
|
||||
import io.kestra.core.services.LabelService;
|
||||
import io.kestra.core.test.flow.TaskFixture;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import jakarta.annotation.Nullable;
|
||||
@@ -132,7 +133,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
public List<Label> getLabels() {
|
||||
return Optional.ofNullable(this.labels).orElse(new ArrayList<>());
|
||||
return ListUtils.emptyOnNull(this.labels);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -156,6 +157,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
.flowRevision(flow.getRevision())
|
||||
.state(new State())
|
||||
.scheduleDate(scheduleDate.map(ChronoZonedDateTime::toInstant).orElse(null))
|
||||
.variables(flow.getVariables())
|
||||
.build();
|
||||
|
||||
List<Label> executionLabels = new ArrayList<>(LabelService.labelsExcludingSystem(flow));
|
||||
@@ -176,8 +178,22 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Customization of Lombok-generated builder.
|
||||
*/
|
||||
public static class ExecutionBuilder {
|
||||
|
||||
/**
|
||||
* Enforce unique values of {@link Label} when using the builder.
|
||||
*
|
||||
* @param labels The labels.
|
||||
* @return Deduplicated labels.
|
||||
*/
|
||||
public ExecutionBuilder labels(List<Label> labels) {
|
||||
this.labels = Label.deduplicate(labels);
|
||||
return this;
|
||||
}
|
||||
|
||||
void prebuild() {
|
||||
this.originalId = this.id;
|
||||
this.metadata = ExecutionMetadata.builder()
|
||||
@@ -225,7 +241,6 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
public Execution withLabels(List<Label> labels) {
|
||||
|
||||
return new Execution(
|
||||
this.tenantId,
|
||||
this.id,
|
||||
@@ -235,7 +250,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
this.taskRunList,
|
||||
this.inputs,
|
||||
this.outputs,
|
||||
labels,
|
||||
Label.deduplicate(labels),
|
||||
this.variables,
|
||||
this.state,
|
||||
this.parentId,
|
||||
@@ -365,7 +380,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
*
|
||||
* @param resolvedTasks normal tasks
|
||||
* @param resolvedErrors errors tasks
|
||||
* @param resolvedErrors finally tasks
|
||||
* @param resolvedFinally finally tasks
|
||||
* @return the flow we need to follow
|
||||
*/
|
||||
public List<ResolvedTask> findTaskDependingFlowState(
|
||||
|
||||
@@ -7,6 +7,7 @@ import io.kestra.core.models.tasks.ResolvedTask;
|
||||
import io.kestra.core.models.tasks.retrys.AbstractRetry;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.*;
|
||||
@@ -62,6 +63,11 @@ public class TaskRun implements TenantInterface {
|
||||
@With
|
||||
Boolean dynamic;
|
||||
|
||||
// Set it to true to force execution even if the execution is killed
|
||||
@Nullable
|
||||
@With
|
||||
Boolean forceExecution;
|
||||
|
||||
@Deprecated
|
||||
public void setItems(String items) {
|
||||
// no-op for backward compatibility
|
||||
@@ -81,7 +87,8 @@ public class TaskRun implements TenantInterface {
|
||||
this.outputs,
|
||||
this.state.withState(state),
|
||||
this.iteration,
|
||||
this.dynamic
|
||||
this.dynamic,
|
||||
this.forceExecution
|
||||
);
|
||||
}
|
||||
|
||||
@@ -99,7 +106,8 @@ public class TaskRun implements TenantInterface {
|
||||
this.outputs,
|
||||
newState,
|
||||
this.iteration,
|
||||
this.dynamic
|
||||
this.dynamic,
|
||||
this.forceExecution
|
||||
);
|
||||
}
|
||||
|
||||
@@ -121,7 +129,8 @@ public class TaskRun implements TenantInterface {
|
||||
this.outputs,
|
||||
this.state.withState(State.Type.FAILED),
|
||||
this.iteration,
|
||||
this.dynamic
|
||||
this.dynamic,
|
||||
this.forceExecution
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
package io.kestra.core.models.flows;
|
||||
|
||||
import io.micronaut.core.annotation.Introspected;
|
||||
import jakarta.validation.constraints.Min;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Positive;
|
||||
|
||||
@SuperBuilder
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Introspected
|
||||
public class Concurrency {
|
||||
@Positive
|
||||
@Min(1)
|
||||
@NotNull
|
||||
private Integer limit;
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ import io.kestra.core.models.tasks.retrys.AbstractRetry;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.models.validations.ManualConstraintViolation;
|
||||
import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.kestra.core.services.FlowService;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
import io.kestra.core.validations.FlowValidation;
|
||||
import io.micronaut.core.annotation.Introspected;
|
||||
@@ -30,8 +29,6 @@ import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
@@ -187,19 +184,32 @@ public class Flow extends AbstractFlow implements HasUID {
|
||||
.toList();
|
||||
}
|
||||
|
||||
public List<Task> allErrorsWithChilds() {
|
||||
public List<Task> allErrorsWithChildren() {
|
||||
var allErrors = allTasksWithChilds().stream()
|
||||
.filter(task -> task.isFlowable() && ((FlowableTask<?>) task).getErrors() != null)
|
||||
.flatMap(task -> ((FlowableTask<?>) task).getErrors().stream())
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
|
||||
if (this.getErrors() != null && !this.getErrors().isEmpty()) {
|
||||
if (!ListUtils.isEmpty(this.getErrors())) {
|
||||
allErrors.addAll(this.getErrors());
|
||||
}
|
||||
|
||||
return allErrors;
|
||||
}
|
||||
|
||||
public List<Task> allFinallyWithChildren() {
|
||||
var allFinally = allTasksWithChilds().stream()
|
||||
.filter(task -> task.isFlowable() && ((FlowableTask<?>) task).getFinally() != null)
|
||||
.flatMap(task -> ((FlowableTask<?>) task).getFinally().stream())
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
|
||||
if (!ListUtils.isEmpty(this.getFinally())) {
|
||||
allFinally.addAll(this.getFinally());
|
||||
}
|
||||
|
||||
return allFinally;
|
||||
}
|
||||
|
||||
public Task findParentTasksByTaskId(String taskId) {
|
||||
return allTasksWithChilds()
|
||||
.stream()
|
||||
|
||||
@@ -3,6 +3,7 @@ package io.kestra.core.models.flows;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.micronaut.core.annotation.Introspected;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
@@ -253,9 +254,22 @@ public class State {
|
||||
return this == Type.KILLED;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return states that are terminal to an execution
|
||||
*/
|
||||
public static List<Type> terminatedTypes() {
|
||||
return Stream.of(Type.values()).filter(type -> type.isTerminated()).toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the final 'failure' of a task depending on <code>allowFailure</code> and <code>allowWarning</code>:
|
||||
* - if both are true -> SUCCESS
|
||||
* - if only <code>allowFailure</code> is true -> WARNING
|
||||
* - if none -> FAILED
|
||||
*/
|
||||
public static State.Type fail(Task task) {
|
||||
return task.isAllowFailure() ? (task.isAllowWarning() ? State.Type.SUCCESS : State.Type.WARNING) : State.Type.FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
@Value
|
||||
|
||||
@@ -108,7 +108,7 @@ public class MultiselectInput extends Input<List<String>> implements ItemTypeInt
|
||||
private List<String> renderExpressionValues(final Function<String, Object> renderer) {
|
||||
Object result;
|
||||
try {
|
||||
result = renderer.apply(expression);
|
||||
result = renderer.apply(expression.trim());
|
||||
} catch (Exception e) {
|
||||
throw ManualConstraintViolation.toConstraintViolationException(
|
||||
"Cannot render 'expression'. Cause: " + e.getMessage(),
|
||||
|
||||
@@ -86,7 +86,7 @@ public class SelectInput extends Input<String> implements RenderableInput {
|
||||
private List<String> renderExpressionValues(final Function<String, Object> renderer) {
|
||||
Object result;
|
||||
try {
|
||||
result = renderer.apply(expression);
|
||||
result = renderer.apply(expression.trim());
|
||||
} catch (Exception e) {
|
||||
throw ManualConstraintViolation.toConstraintViolationException(
|
||||
"Cannot render 'expression'. Cause: " + e.getMessage(),
|
||||
|
||||
@@ -30,7 +30,7 @@ import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
* Helper class for task runners and script tasks.
|
||||
*/
|
||||
public final class ScriptService {
|
||||
private static final Pattern INTERNAL_STORAGE_PATTERN = Pattern.compile("(kestra:\\/\\/[-a-zA-Z0-9%._\\+~#=/]*)");
|
||||
private static final Pattern INTERNAL_STORAGE_PATTERN = Pattern.compile("(kestra:\\/\\/[-\\p{Alnum}._\\+~#=/]*)", Pattern.UNICODE_CHARACTER_CLASS);
|
||||
|
||||
// These are the three common additional variables task runners must provide for variable rendering.
|
||||
public static final String VAR_WORKING_DIR = "workingDir";
|
||||
|
||||
@@ -329,6 +329,14 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
pluginClassByIdentifier.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
**/
|
||||
@Override
|
||||
public boolean isVersioningSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public record PluginBundleIdentifier(@Nullable URL location) {
|
||||
|
||||
public static PluginBundleIdentifier CORE = new PluginBundleIdentifier(null);
|
||||
|
||||
@@ -151,7 +151,7 @@ public class LocalPluginManager implements PluginManager {
|
||||
* {@inheritDoc}
|
||||
**/
|
||||
@Override
|
||||
public PluginArtifact install(File file, boolean installForRegistration, @Nullable Path localRepositoryPath) {
|
||||
public PluginArtifact install(File file, boolean installForRegistration, @Nullable Path localRepositoryPath, boolean forceInstallOnExistingVersions) {
|
||||
try {
|
||||
PluginArtifact artifact = PluginArtifact.fromFile(file);
|
||||
log.info("Installing managed plugin artifact '{}'", artifact);
|
||||
|
||||
@@ -55,14 +55,16 @@ public interface PluginManager extends AutoCloseable {
|
||||
/**
|
||||
* Installs the given plugin artifact.
|
||||
*
|
||||
* @param file the plugin JAR file.
|
||||
* @param installForRegistration specify whether plugin artifacts should be scanned and registered.
|
||||
* @param localRepositoryPath the optional local repository path to install artifact.
|
||||
* @param file the plugin JAR file.
|
||||
* @param installForRegistration specify whether plugin artifacts should be scanned and registered.
|
||||
* @param localRepositoryPath the optional local repository path to install artifact.
|
||||
* @param forceInstallOnExistingVersions specify whether plugin should be forced install upon the existing one
|
||||
* @return The URI of the installed plugin.
|
||||
*/
|
||||
PluginArtifact install(final File file,
|
||||
boolean installForRegistration,
|
||||
@Nullable Path localRepositoryPath);
|
||||
@Nullable Path localRepositoryPath,
|
||||
boolean forceInstallOnExistingVersions);
|
||||
|
||||
/**
|
||||
* Installs the given plugin artifact.
|
||||
|
||||
@@ -116,4 +116,11 @@ public interface PluginRegistry {
|
||||
default void clear() {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether plugin-versioning is supported by this registry.
|
||||
*
|
||||
* @return {@code true} if supported. Otherwise {@code false}.
|
||||
*/
|
||||
boolean isVersioningSupported();
|
||||
}
|
||||
|
||||
@@ -18,9 +18,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileSystemNotFoundException;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@@ -202,19 +204,13 @@ public class PluginScanner {
|
||||
|
||||
var guidesDirectory = classLoader.getResource("doc/guides");
|
||||
if (guidesDirectory != null) {
|
||||
try (var fileSystem = FileSystems.newFileSystem(guidesDirectory.toURI(), Collections.emptyMap())) {
|
||||
var root = fileSystem.getPath("/doc/guides");
|
||||
try (var stream = Files.walk(root, 1)) {
|
||||
stream
|
||||
.skip(1) // first element is the root element
|
||||
.sorted(Comparator.comparing(path -> path.getName(path.getParent().getNameCount()).toString()))
|
||||
.forEach(guide -> {
|
||||
var guideName = guide.getName(guide.getParent().getNameCount()).toString();
|
||||
guides.add(guideName.substring(0, guideName.lastIndexOf('.')));
|
||||
});
|
||||
}
|
||||
try {
|
||||
var root = Path.of(guidesDirectory.toURI());
|
||||
addGuides(root, guides);
|
||||
} catch (IOException | URISyntaxException e) {
|
||||
// silently fail
|
||||
} catch (FileSystemNotFoundException e) {
|
||||
addGuidesThroughNewFileSystem(guidesDirectory, guides);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,6 +239,27 @@ public class PluginScanner {
|
||||
.build();
|
||||
}
|
||||
|
||||
private static void addGuidesThroughNewFileSystem(URL guidesDirectory, List<String> guides) {
|
||||
try (var fileSystem = FileSystems.newFileSystem(guidesDirectory.toURI(), Collections.emptyMap())) {
|
||||
var root = fileSystem.getPath("doc/guides");
|
||||
addGuides(root, guides);
|
||||
} catch (IOException | URISyntaxException e) {
|
||||
// silently fail
|
||||
}
|
||||
}
|
||||
|
||||
private static void addGuides(Path root, List<String> guides) throws IOException {
|
||||
try (var stream = Files.walk(root, 1)) {
|
||||
stream
|
||||
.skip(1) // first element is the root element
|
||||
.sorted(Comparator.comparing(path -> path.getName(path.getParent().getNameCount()).toString()))
|
||||
.forEach(guide -> {
|
||||
var guideName = guide.getName(guide.getParent().getNameCount()).toString();
|
||||
guides.add(guideName.substring(0, guideName.lastIndexOf('.')));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static Manifest getManifest(ClassLoader classLoader) {
|
||||
try {
|
||||
URL url = classLoader.getResource(JarFile.MANIFEST_NAME);
|
||||
|
||||
@@ -86,7 +86,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
DeserializationContext context) throws IOException {
|
||||
Class<? extends Plugin> pluginType = null;
|
||||
|
||||
final String identifier = extractPluginRawIdentifier(node);
|
||||
final String identifier = extractPluginRawIdentifier(node, pluginRegistry.isVersioningSupported());
|
||||
if (identifier != null) {
|
||||
log.trace("Looking for Plugin for: {}",
|
||||
identifier
|
||||
@@ -103,7 +103,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
);
|
||||
|
||||
if (DataChart.class.isAssignableFrom(pluginType)) {
|
||||
final Class<? extends Plugin> dataFilterClass = pluginRegistry.findClassByIdentifier(extractPluginRawIdentifier(node.get("data")));
|
||||
final Class<? extends Plugin> dataFilterClass = pluginRegistry.findClassByIdentifier(extractPluginRawIdentifier(node.get("data"), pluginRegistry.isVersioningSupported()));
|
||||
ParameterizedType genericDataFilterClass = (ParameterizedType) dataFilterClass.getGenericSuperclass();
|
||||
Type dataFieldsEnum = genericDataFilterClass.getActualTypeArguments()[0];
|
||||
TypeFactory typeFactory = JacksonMapper.ofJson().getTypeFactory();
|
||||
@@ -142,7 +142,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
);
|
||||
}
|
||||
|
||||
static String extractPluginRawIdentifier(final JsonNode node) {
|
||||
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
|
||||
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
|
||||
|
||||
@@ -150,6 +150,6 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
return null;
|
||||
}
|
||||
|
||||
return version != null && !version.isEmpty() ? type + ":" + version : type;
|
||||
return isVersioningSupported && version != null && !version.isEmpty() ? type + ":" + version : type;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ public interface QueueFactoryInterface {
|
||||
String SUBFLOWEXECUTIONRESULT_NAMED = "subflowExecutionResultQueue";
|
||||
String CLUSTER_EVENT_NAMED = "clusterEventQueue";
|
||||
String SUBFLOWEXECUTIONEND_NAMED = "subflowExecutionEndQueue";
|
||||
String EXECUTION_RUNNING_NAMED = "executionRunningQueue";
|
||||
|
||||
QueueInterface<Execution> execution();
|
||||
|
||||
@@ -62,4 +63,6 @@ public interface QueueFactoryInterface {
|
||||
QueueInterface<SubflowExecutionResult> subflowExecutionResult();
|
||||
|
||||
QueueInterface<SubflowExecutionEnd> subflowExecutionEnd();
|
||||
|
||||
QueueInterface<ExecutionRunning> executionRunning();
|
||||
}
|
||||
|
||||
@@ -27,8 +27,6 @@ public class QueueService {
|
||||
return ((Executor) object).getExecution().getId();
|
||||
} else if (object.getClass() == MetricEntry.class) {
|
||||
return null;
|
||||
} else if (object.getClass() == ExecutionRunning.class) {
|
||||
return ((ExecutionRunning) object).getExecution().getId();
|
||||
} else if (object.getClass() == SubflowExecutionEnd.class) {
|
||||
return ((SubflowExecutionEnd) object).getParentExecutionId();
|
||||
} else {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
@@ -11,7 +12,7 @@ import lombok.With;
|
||||
@Value
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class ExecutionRunning {
|
||||
public class ExecutionRunning implements HasUID {
|
||||
String tenantId;
|
||||
|
||||
@NotNull
|
||||
@@ -26,6 +27,7 @@ public class ExecutionRunning {
|
||||
@With
|
||||
ConcurrencyState concurrencyState;
|
||||
|
||||
@Override
|
||||
public String uid() {
|
||||
return IdUtils.fromPartsAndSeparator('|', this.tenantId, this.namespace, this.flowId, this.execution.getId());
|
||||
}
|
||||
|
||||
@@ -67,6 +67,9 @@ public class ExecutorService {
|
||||
@Inject
|
||||
private WorkerGroupExecutorInterface workerGroupExecutorInterface;
|
||||
|
||||
@Inject
|
||||
private WorkerJobRunningStateStore workerJobRunningStateStore;
|
||||
|
||||
protected FlowMetaStoreInterface flowExecutorInterface;
|
||||
|
||||
@Inject
|
||||
@@ -97,49 +100,39 @@ public class ExecutorService {
|
||||
return this.flowExecutorInterface;
|
||||
}
|
||||
|
||||
public Executor checkConcurrencyLimit(Executor executor, FlowInterface flow, Execution execution, long count) {
|
||||
// if above the limit, handle concurrency limit based on its behavior
|
||||
if (count >= flow.getConcurrency().getLimit()) {
|
||||
public ExecutionRunning processExecutionRunning(FlowInterface flow, int runningCount, ExecutionRunning executionRunning) {
|
||||
// if concurrency was removed, it can be null as we always get the latest flow definition
|
||||
if (flow.getConcurrency() != null && runningCount >= flow.getConcurrency().getLimit()) {
|
||||
return switch (flow.getConcurrency().getBehavior()) {
|
||||
case QUEUE -> {
|
||||
var newExecution = execution.withState(State.Type.QUEUED);
|
||||
|
||||
ExecutionRunning executionRunning = ExecutionRunning.builder()
|
||||
.tenantId(flow.getTenantId())
|
||||
.namespace(flow.getNamespace())
|
||||
.flowId(flow.getId())
|
||||
.execution(newExecution)
|
||||
.concurrencyState(ExecutionRunning.ConcurrencyState.QUEUED)
|
||||
.build();
|
||||
|
||||
// when max concurrency is reached, we throttle the execution and stop processing
|
||||
logService.logExecution(
|
||||
newExecution,
|
||||
executionRunning.getExecution(),
|
||||
Level.INFO,
|
||||
"Flow is queued due to concurrency limit exceeded, {} running(s)",
|
||||
count
|
||||
"Execution is queued due to concurrency limit exceeded, {} running(s)",
|
||||
runningCount
|
||||
);
|
||||
// return the execution queued
|
||||
yield executor
|
||||
.withExecutionRunning(executionRunning)
|
||||
.withExecution(newExecution, "checkConcurrencyLimit");
|
||||
var newExecution = executionRunning.getExecution().withState(State.Type.QUEUED);
|
||||
metricRegistry.counter(MetricRegistry.METRIC_EXECUTOR_EXECUTION_QUEUED_COUNT, MetricRegistry.METRIC_EXECUTOR_EXECUTION_QUEUED_COUNT_DESCRIPTION, metricRegistry.tags(newExecution)).increment();
|
||||
yield executionRunning
|
||||
.withExecution(newExecution)
|
||||
.withConcurrencyState(ExecutionRunning.ConcurrencyState.QUEUED);
|
||||
}
|
||||
case CANCEL ->
|
||||
executor.withExecution(execution.withState(State.Type.CANCELLED), "checkConcurrencyLimit");
|
||||
executionRunning
|
||||
.withExecution(executionRunning.getExecution().withState(State.Type.CANCELLED))
|
||||
.withConcurrencyState(ExecutionRunning.ConcurrencyState.RUNNING);
|
||||
case FAIL ->
|
||||
executor.withException(new IllegalStateException("Flow is FAILED due to concurrency limit exceeded"), "checkConcurrencyLimit");
|
||||
executionRunning
|
||||
.withExecution(executionRunning.getExecution().failedExecutionFromExecutor(new IllegalStateException("Execution is FAILED due to concurrency limit exceeded")).getExecution())
|
||||
.withConcurrencyState(ExecutionRunning.ConcurrencyState.RUNNING);
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
// if under the limit, update the executor with a RUNNING ExecutionRunning to track them
|
||||
var executionRunning = new ExecutionRunning(
|
||||
flow.getTenantId(),
|
||||
flow.getNamespace(),
|
||||
flow.getId(),
|
||||
executor.getExecution(),
|
||||
ExecutionRunning.ConcurrencyState.RUNNING
|
||||
);
|
||||
return executor.withExecutionRunning(executionRunning);
|
||||
// if under the limit, run it!
|
||||
return executionRunning
|
||||
.withExecution(executionRunning.getExecution().withState(State.Type.RUNNING))
|
||||
.withConcurrencyState(ExecutionRunning.ConcurrencyState.RUNNING);
|
||||
}
|
||||
|
||||
public Executor process(Executor executor) {
|
||||
@@ -664,7 +657,7 @@ public class ExecutorService {
|
||||
.taskRunId(workerTaskResult.getTaskRun().getId())
|
||||
.executionId(executor.getExecution().getId())
|
||||
.date(workerTaskResult.getTaskRun().getState().maxDate().plus(duration != null ? duration : timeout))
|
||||
.state(duration != null ? behavior.mapToState() : State.Type.FAILED)
|
||||
.state(duration != null ? behavior.mapToState() : State.Type.fail(pauseTask))
|
||||
.delayType(ExecutionDelay.DelayType.RESUME_FLOW)
|
||||
.build();
|
||||
}
|
||||
@@ -732,6 +725,7 @@ public class ExecutorService {
|
||||
List<TaskRun> afterExecutionNexts = FlowableUtils.resolveSequentialNexts(executor.getExecution(), afterExecutionResolvedTasks)
|
||||
.stream()
|
||||
.map(throwFunction(NextTaskRun::getTaskRun))
|
||||
.map(taskRun -> taskRun.withForceExecution(true)) // forceExecution so it would be executed even if the execution is killed
|
||||
.toList();
|
||||
if (!afterExecutionNexts.isEmpty()) {
|
||||
return executor.withTaskRun(afterExecutionNexts, "handleAfterExecution ");
|
||||
@@ -1072,6 +1066,25 @@ public class ExecutorService {
|
||||
newExecution = executionService.killParentTaskruns(taskRun, newExecution);
|
||||
}
|
||||
executor.withExecution(newExecution, "addWorkerTaskResult");
|
||||
if (taskRun.getState().isTerminated()) {
|
||||
log.trace("TaskRun terminated: {}", taskRun);
|
||||
workerJobRunningStateStore.deleteByKey(taskRun.getId());
|
||||
metricRegistry
|
||||
.counter(
|
||||
MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_COUNT,
|
||||
MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_COUNT_DESCRIPTION,
|
||||
metricRegistry.tags(workerTaskResult)
|
||||
)
|
||||
.increment();
|
||||
|
||||
metricRegistry
|
||||
.timer(
|
||||
MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_DURATION,
|
||||
MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_DURATION_DESCRIPTION,
|
||||
metricRegistry.tags(workerTaskResult)
|
||||
)
|
||||
.record(taskRun.getState().getDuration());
|
||||
}
|
||||
}
|
||||
|
||||
// Note: as the flow is only used in an error branch and it can take time to load, we pass it thought a Supplier
|
||||
|
||||
@@ -286,18 +286,10 @@ public class FlowableUtils {
|
||||
|
||||
// start as many tasks as we have concurrency slots
|
||||
return collect.values().stream()
|
||||
.map(resolvedTasks -> filterCreated(resolvedTasks, taskRuns, parentTaskRun))
|
||||
.map(resolvedTasks -> resolveSequentialNexts(execution, resolvedTasks, null, null, parentTaskRun))
|
||||
.filter(resolvedTasks -> !resolvedTasks.isEmpty())
|
||||
.limit(concurrencySlots)
|
||||
.map(resolvedTasks -> resolvedTasks.getFirst().toNextTaskRun(execution))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static List<ResolvedTask> filterCreated(List<ResolvedTask> tasks, List<TaskRun> taskRuns, TaskRun parentTaskRun) {
|
||||
return tasks.stream()
|
||||
.filter(resolvedTask -> taskRuns.stream()
|
||||
.noneMatch(taskRun -> FlowableUtils.isTaskRunFor(resolvedTask, taskRun, parentTaskRun))
|
||||
)
|
||||
.map(resolvedTasks -> resolvedTasks.getFirst())
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import io.kestra.core.models.flows.input.SecretInput;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
import io.kestra.plugin.core.trigger.Schedule;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.With;
|
||||
|
||||
@@ -27,6 +28,7 @@ import java.util.function.Consumer;
|
||||
*/
|
||||
public final class RunVariables {
|
||||
public static final String SECRET_CONSUMER_VARIABLE_NAME = "addSecretConsumer";
|
||||
public static final String FIXTURE_FILES_KEY = "io.kestra.datatype:test_fixtures_files";
|
||||
|
||||
/**
|
||||
* Creates an immutable map representation of the given {@link Task}.
|
||||
@@ -181,9 +183,6 @@ public final class RunVariables {
|
||||
// Flow
|
||||
if (flow != null) {
|
||||
builder.put("flow", RunVariables.of(flow));
|
||||
if (flow.getVariables() != null) {
|
||||
builder.put("vars", flow.getVariables());
|
||||
}
|
||||
}
|
||||
|
||||
// Task
|
||||
@@ -298,16 +297,19 @@ public final class RunVariables {
|
||||
|
||||
if (execution.getTrigger() != null && execution.getTrigger().getVariables() != null) {
|
||||
builder.put("trigger", execution.getTrigger().getVariables());
|
||||
|
||||
// temporal hack to add back the `schedule`variables
|
||||
// will be removed in 2.0
|
||||
if (Schedule.class.getName().equals(execution.getTrigger().getType())) {
|
||||
// add back its variables inside the `schedule` variables
|
||||
builder.put("schedule", execution.getTrigger().getVariables());
|
||||
}
|
||||
}
|
||||
|
||||
if (execution.getLabels() != null) {
|
||||
builder.put("labels", Label.toNestedMap(execution.getLabels()));
|
||||
}
|
||||
|
||||
if (execution.getVariables() != null) {
|
||||
builder.putAll(execution.getVariables());
|
||||
}
|
||||
|
||||
if (flow == null) {
|
||||
Flow flowFromExecution = Flow.builder()
|
||||
.id(execution.getFlowId())
|
||||
@@ -319,6 +321,20 @@ public final class RunVariables {
|
||||
}
|
||||
}
|
||||
|
||||
// variables
|
||||
Optional.ofNullable(execution)
|
||||
.map(Execution::getVariables)
|
||||
.or(() -> Optional.ofNullable(flow).map(FlowInterface::getVariables))
|
||||
.map(HashMap::new)
|
||||
.ifPresent(variables -> {
|
||||
Object fixtureFiles = variables.remove(FIXTURE_FILES_KEY);
|
||||
builder.put("vars", ImmutableMap.copyOf(variables));
|
||||
|
||||
if (fixtureFiles != null) {
|
||||
builder.put("files", fixtureFiles);
|
||||
}
|
||||
});
|
||||
|
||||
// Kestra configuration
|
||||
if (kestraConfiguration != null) {
|
||||
Map<String, String> kestra = HashMap.newHashMap(2);
|
||||
|
||||
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.exceptions.DeserializationException;
|
||||
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.metrics.MetricRegistry;
|
||||
@@ -19,6 +18,7 @@ import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.kestra.core.server.*;
|
||||
import io.kestra.core.services.LabelService;
|
||||
import io.kestra.core.services.LogService;
|
||||
import io.kestra.core.services.MaintenanceService;
|
||||
import io.kestra.core.services.VariablesService;
|
||||
import io.kestra.core.services.WorkerGroupService;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
@@ -158,6 +158,9 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
private TracerFactory tracerFactory;
|
||||
private Tracer tracer;
|
||||
|
||||
@Inject
|
||||
private MaintenanceService maintenanceService;
|
||||
|
||||
/**
|
||||
* Creates a new {@link Worker} instance.
|
||||
*
|
||||
@@ -285,8 +288,12 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
));
|
||||
|
||||
this.clusterEventQueue.ifPresent(clusterEventQueueInterface -> this.receiveCancellations.addFirst(clusterEventQueueInterface.receive(this::clusterEventQueue)));
|
||||
if (this.maintenanceService.isInMaintenanceMode()) {
|
||||
enterMaintenance();
|
||||
} else {
|
||||
setState(ServiceState.RUNNING);
|
||||
}
|
||||
|
||||
setState(ServiceState.RUNNING);
|
||||
if (workerGroupKey != null) {
|
||||
log.info("Worker started with {} thread(s) in group '{}'", numThreads, workerGroupKey);
|
||||
}
|
||||
@@ -304,21 +311,25 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
ClusterEvent clusterEvent = either.getLeft();
|
||||
log.info("Cluster event received: {}", clusterEvent);
|
||||
switch (clusterEvent.eventType()) {
|
||||
case MAINTENANCE_ENTER -> {
|
||||
this.executionKilledQueue.pause();
|
||||
this.workerJobQueue.pause();
|
||||
|
||||
this.setState(ServiceState.MAINTENANCE);
|
||||
}
|
||||
case MAINTENANCE_EXIT -> {
|
||||
this.executionKilledQueue.resume();
|
||||
this.workerJobQueue.resume();
|
||||
|
||||
this.setState(ServiceState.RUNNING);
|
||||
}
|
||||
case MAINTENANCE_ENTER -> enterMaintenance();
|
||||
case MAINTENANCE_EXIT -> exitMaintenance();
|
||||
}
|
||||
}
|
||||
|
||||
private void enterMaintenance() {
|
||||
this.executionKilledQueue.pause();
|
||||
this.workerJobQueue.pause();
|
||||
|
||||
this.setState(ServiceState.MAINTENANCE);
|
||||
}
|
||||
|
||||
private void exitMaintenance() {
|
||||
this.executionKilledQueue.resume();
|
||||
this.workerJobQueue.resume();
|
||||
|
||||
this.setState(ServiceState.RUNNING);
|
||||
}
|
||||
|
||||
private void setState(final ServiceState state) {
|
||||
this.state.set(state);
|
||||
Map<String, Object> properties = new HashMap<>();
|
||||
@@ -395,11 +406,16 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
} catch (IllegalVariableEvaluationException e) {
|
||||
RunContextLogger contextLogger = runContextLoggerFactory.create(currentWorkerTask);
|
||||
contextLogger.logger().error("Failed evaluating runIf: {}", e.getMessage(), e);
|
||||
try {
|
||||
this.workerTaskResultQueue.emit(new WorkerTaskResult(workerTask.fail()));
|
||||
} catch (QueueException ex) {
|
||||
log.error("Unable to emit the worker task result for task {} taskrun {}", currentWorkerTask.getTask().getId(), currentWorkerTask.getTaskRun().getId(), e);
|
||||
}
|
||||
} catch (QueueException e) {
|
||||
log.error("Unable to emit the worker task result for task {} taskrun {}", currentWorkerTask.getTask().getId(), currentWorkerTask.getTaskRun().getId(), e);
|
||||
}
|
||||
|
||||
if (workerTaskResult.getTaskRun().getState().isFailed() && !currentWorkerTask.getTask().isAllowFailure()) {
|
||||
if (workerTaskResult == null || workerTaskResult.getTaskRun().getState().isFailed() && !currentWorkerTask.getTask().isAllowFailure()) {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -624,7 +640,7 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
));
|
||||
}
|
||||
|
||||
if (killedExecution.contains(workerTask.getTaskRun().getExecutionId())) {
|
||||
if (! Boolean.TRUE.equals(workerTask.getTaskRun().getForceExecution()) && killedExecution.contains(workerTask.getTaskRun().getExecutionId())) {
|
||||
WorkerTaskResult workerTaskResult = new WorkerTaskResult(workerTask.getTaskRun().withState(KILLED));
|
||||
try {
|
||||
this.workerTaskResultQueue.emit(workerTaskResult);
|
||||
@@ -776,7 +792,7 @@ public class Worker implements Service, Runnable, AutoCloseable {
|
||||
|
||||
if (!(workerTask.getTask() instanceof RunnableTask<?> task)) {
|
||||
// This should never happen but better to deal with it than crashing the Worker
|
||||
var state = workerTask.getTask().isAllowFailure() ? workerTask.getTask().isAllowWarning() ? SUCCESS : WARNING : FAILED;
|
||||
var state = State.Type.fail(workerTask.getTask());
|
||||
TaskRunAttempt attempt = TaskRunAttempt.builder()
|
||||
.state(new io.kestra.core.models.flows.State().withState(state))
|
||||
.workerId(this.id)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
/**
|
||||
* State store containing all workers' jobs in RUNNING state.
|
||||
*
|
||||
* @see WorkerJob
|
||||
*/
|
||||
public interface WorkerJobRunningStateStore {
|
||||
|
||||
/**
|
||||
* Deletes a running worker job for the given key.
|
||||
*
|
||||
* <p>
|
||||
* A key can be a {@link WorkerTask} Task Run ID.
|
||||
* </p>
|
||||
*
|
||||
* @param key the key of the worker job to be deleted.
|
||||
*/
|
||||
void deleteByKey(String key);
|
||||
}
|
||||
@@ -48,7 +48,7 @@ public class WorkerTask extends WorkerJob {
|
||||
* @return this worker task, updated
|
||||
*/
|
||||
public TaskRun fail() {
|
||||
var state = this.task.isAllowFailure() ? this.task.isAllowWarning() ? State.Type.SUCCESS : State.Type.WARNING : State.Type.FAILED;
|
||||
var state = State.Type.fail(task);
|
||||
return this.getTaskRun().withState(state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ public class DateAddFilter extends AbstractDate implements Filter {
|
||||
return null;
|
||||
}
|
||||
|
||||
final Long amount = (Long) args.get("amount");
|
||||
final Long amount = getAsLong(args.get("amount"), lineNumber, self);
|
||||
final String unit = (String) args.get("unit");
|
||||
final String timeZone = (String) args.get("timeZone");
|
||||
final String existingFormat = (String) args.get("existingFormat");
|
||||
@@ -36,4 +36,24 @@ public class DateAddFilter extends AbstractDate implements Filter {
|
||||
|
||||
return format(plus, args, context);
|
||||
}
|
||||
|
||||
public static Long getAsLong(Object value, int lineNumber, PebbleTemplate self) {
|
||||
if (value instanceof Long longValue) {
|
||||
return longValue;
|
||||
} else if (value instanceof Integer integerValue) {
|
||||
return integerValue.longValue();
|
||||
} else if (value instanceof Number numberValue) {
|
||||
return numberValue.longValue();
|
||||
} else if (value instanceof String stringValue) {
|
||||
try {
|
||||
return Long.parseLong(stringValue);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new PebbleException(e, "%s can't be converted to long".formatted(stringValue),
|
||||
lineNumber, self != null ? self.getName() : "Unknown");
|
||||
}
|
||||
}
|
||||
throw new PebbleException(null, "Incorrect %s format, must be a number".formatted(value),
|
||||
lineNumber, self != null ? self.getName() : "Unknown");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
private final LogService logService;
|
||||
protected SchedulerExecutionStateInterface executionState;
|
||||
private final WorkerGroupExecutorInterface workerGroupExecutorInterface;
|
||||
private final MaintenanceService maintenanceService;
|
||||
|
||||
// must be volatile as it's updated by the flow listener thread and read by the scheduleExecutor thread
|
||||
private volatile Boolean isReady = false;
|
||||
@@ -136,6 +137,8 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
this.serviceStateEventPublisher = applicationContext.getBean(ApplicationEventPublisher.class);
|
||||
this.executionEventPublisher = applicationContext.getBean(ApplicationEventPublisher.class);
|
||||
this.workerGroupExecutorInterface = applicationContext.getBean(WorkerGroupExecutorInterface.class);
|
||||
this.maintenanceService = applicationContext.getBean(MaintenanceService.class);
|
||||
|
||||
setState(ServiceState.CREATED);
|
||||
}
|
||||
|
||||
@@ -289,8 +292,11 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
|
||||
// listen to cluster events
|
||||
this.clusterEventQueue.ifPresent(clusterEventQueueInterface -> this.receiveCancellations.addFirst(((QueueInterface<ClusterEvent>) clusterEventQueueInterface).receive(this::clusterEventQueue)));
|
||||
|
||||
setState(ServiceState.RUNNING);
|
||||
if (this.maintenanceService.isInMaintenanceMode()) {
|
||||
enterMaintenance();
|
||||
} else {
|
||||
setState(ServiceState.RUNNING);
|
||||
}
|
||||
log.info("Scheduler started");
|
||||
}
|
||||
|
||||
@@ -399,31 +405,35 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
ClusterEvent clusterEvent = either.getLeft();
|
||||
log.info("Cluster event received: {}", clusterEvent);
|
||||
switch (clusterEvent.eventType()) {
|
||||
case MAINTENANCE_ENTER -> {
|
||||
this.executionQueue.pause();
|
||||
this.triggerQueue.pause();
|
||||
this.workerJobQueue.pause();
|
||||
this.workerTriggerResultQueue.pause();
|
||||
this.executionKilledQueue.pause();
|
||||
this.pauseAdditionalQueues();
|
||||
|
||||
this.isPaused.set(true);
|
||||
this.setState(ServiceState.MAINTENANCE);
|
||||
}
|
||||
case MAINTENANCE_EXIT -> {
|
||||
this.executionQueue.resume();
|
||||
this.triggerQueue.resume();
|
||||
this.workerJobQueue.resume();
|
||||
this.workerTriggerResultQueue.resume();
|
||||
this.executionKilledQueue.resume();
|
||||
this.resumeAdditionalQueues();
|
||||
|
||||
this.isPaused.set(false);
|
||||
this.setState(ServiceState.RUNNING);
|
||||
}
|
||||
case MAINTENANCE_ENTER -> enterMaintenance();
|
||||
case MAINTENANCE_EXIT -> exitMaintenance();
|
||||
}
|
||||
}
|
||||
|
||||
private void enterMaintenance() {
|
||||
this.executionQueue.pause();
|
||||
this.triggerQueue.pause();
|
||||
this.workerJobQueue.pause();
|
||||
this.workerTriggerResultQueue.pause();
|
||||
this.executionKilledQueue.pause();
|
||||
this.pauseAdditionalQueues();
|
||||
|
||||
this.isPaused.set(true);
|
||||
this.setState(ServiceState.MAINTENANCE);
|
||||
}
|
||||
|
||||
private void exitMaintenance() {
|
||||
this.executionQueue.resume();
|
||||
this.triggerQueue.resume();
|
||||
this.workerJobQueue.resume();
|
||||
this.workerTriggerResultQueue.resume();
|
||||
this.executionKilledQueue.resume();
|
||||
this.resumeAdditionalQueues();
|
||||
|
||||
this.isPaused.set(false);
|
||||
this.setState(ServiceState.RUNNING);
|
||||
}
|
||||
|
||||
protected void resumeAdditionalQueues() {
|
||||
// by default: do nothing
|
||||
}
|
||||
@@ -996,6 +1006,9 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
}
|
||||
|
||||
setState(ServiceState.TERMINATING);
|
||||
this.receiveCancellations.forEach(Runnable::run);
|
||||
this.scheduleExecutor.shutdown();
|
||||
this.executionMonitorExecutor.shutdown();
|
||||
try {
|
||||
if (onClose != null) {
|
||||
onClose.run();
|
||||
@@ -1003,9 +1016,6 @@ public abstract class AbstractScheduler implements Scheduler, Service {
|
||||
} catch (Exception e) {
|
||||
log.error("Unexpected error while terminating scheduler.", e);
|
||||
}
|
||||
this.receiveCancellations.forEach(Runnable::run);
|
||||
this.scheduleExecutor.shutdown();
|
||||
this.executionMonitorExecutor.shutdown();
|
||||
setState(ServiceState.TERMINATED_GRACEFULLY);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.amazon.ion.IonSystem;
|
||||
import com.amazon.ion.system.*;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.StreamReadConstraints;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
@@ -36,6 +37,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TimeZone;
|
||||
|
||||
import static com.fasterxml.jackson.core.StreamReadConstraints.DEFAULT_MAX_STRING_LEN;
|
||||
|
||||
public final class JacksonMapper {
|
||||
public static final TypeReference<Map<String, Object>> MAP_TYPE_REFERENCE = new TypeReference<>() {};
|
||||
public static final TypeReference<List<Object>> LIST_TYPE_REFERENCE = new TypeReference<>() {};
|
||||
@@ -43,6 +46,12 @@ public final class JacksonMapper {
|
||||
|
||||
private JacksonMapper() {}
|
||||
|
||||
static {
|
||||
StreamReadConstraints.overrideDefaultStreamReadConstraints(
|
||||
StreamReadConstraints.builder().maxNameLength(DEFAULT_MAX_STRING_LEN).build()
|
||||
);
|
||||
}
|
||||
|
||||
private static final ObjectMapper MAPPER = JacksonMapper.configure(
|
||||
new ObjectMapper()
|
||||
);
|
||||
@@ -52,7 +61,7 @@ public final class JacksonMapper {
|
||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
|
||||
public static ObjectMapper ofJson() {
|
||||
return MAPPER;
|
||||
return JacksonMapper.ofJson(true);
|
||||
}
|
||||
|
||||
public static ObjectMapper ofJson(boolean strict) {
|
||||
|
||||
@@ -111,7 +111,7 @@ public interface Service extends AutoCloseable {
|
||||
* </pre>
|
||||
*/
|
||||
enum ServiceState {
|
||||
CREATED(1, 2, 3), // 0
|
||||
CREATED(1, 2, 3, 4, 9), // 0
|
||||
RUNNING(2, 3, 4, 9), // 1
|
||||
ERROR(4), // 2
|
||||
DISCONNECTED(4, 7), // 3
|
||||
|
||||
@@ -215,9 +215,10 @@ public class ServiceLivenessManager extends AbstractServiceLivenessTask {
|
||||
stateLock.lock();
|
||||
// Optional callback to be executed at the end.
|
||||
Runnable returnCallback = null;
|
||||
|
||||
localServiceState = localServiceState(service);
|
||||
try {
|
||||
localServiceState = localServiceState(service);
|
||||
|
||||
|
||||
if (localServiceState == null) {
|
||||
return null; // service has been unregistered.
|
||||
}
|
||||
@@ -266,7 +267,7 @@ public class ServiceLivenessManager extends AbstractServiceLivenessTask {
|
||||
// Update the local instance
|
||||
this.serviceRegistry.register(localServiceState.with(remoteInstance));
|
||||
} catch (Exception e) {
|
||||
final ServiceInstance localInstance = localServiceState(service).instance();
|
||||
final ServiceInstance localInstance = localServiceState.instance();
|
||||
log.error("[Service id={}, type='{}', hostname='{}'] Failed to update state to {}. Error: {}",
|
||||
localInstance.uid(),
|
||||
localInstance.type(),
|
||||
@@ -282,7 +283,7 @@ public class ServiceLivenessManager extends AbstractServiceLivenessTask {
|
||||
returnCallback.run();
|
||||
}
|
||||
}
|
||||
return localServiceState(service).instance();
|
||||
return Optional.ofNullable(localServiceState(service)).map(LocalServiceState::instance).orElse(null);
|
||||
}
|
||||
|
||||
private void mayDisableStateUpdate(final Service service, final ServiceInstance instance) {
|
||||
@@ -336,9 +337,11 @@ public class ServiceLivenessManager extends AbstractServiceLivenessTask {
|
||||
final Service service,
|
||||
final ServiceInstance instance,
|
||||
final boolean isLivenessEnabled) {
|
||||
// Never shutdown STANDALONE server or WEB_SERVER service.
|
||||
if (instance.server().type().equals(ServerInstance.Type.STANDALONE) ||
|
||||
instance.is(ServiceType.WEBSERVER)) {
|
||||
// Never shutdown STANDALONE server or WEBSERVER and INDEXER services.
|
||||
if (ServerInstance.Type.STANDALONE.equals(instance.server().type()) ||
|
||||
instance.is(ServiceType.INDEXER) ||
|
||||
instance.is(ServiceType.WEBSERVER)
|
||||
) {
|
||||
// Force the RUNNING state.
|
||||
return Optional.of(instance.state(Service.ServiceState.RUNNING, now, null));
|
||||
}
|
||||
|
||||
@@ -212,7 +212,16 @@ public class ExecutionService {
|
||||
|
||||
// We need to remove global error tasks and flowable error tasks if any
|
||||
flow
|
||||
.allErrorsWithChilds()
|
||||
.allErrorsWithChildren()
|
||||
.forEach(task -> newTaskRuns.removeIf(taskRun -> taskRun.getTaskId().equals(task.getId())));
|
||||
|
||||
// We need to remove global finally tasks and flowable error tasks if any
|
||||
flow
|
||||
.allFinallyWithChildren()
|
||||
.forEach(task -> newTaskRuns.removeIf(taskRun -> taskRun.getTaskId().equals(task.getId())));
|
||||
|
||||
// We need to remove afterExecution tasks
|
||||
ListUtils.emptyOnNull(flow.getAfterExecution())
|
||||
.forEach(task -> newTaskRuns.removeIf(taskRun -> taskRun.getTaskId().equals(task.getId())));
|
||||
|
||||
// Build and launch new execution
|
||||
|
||||
@@ -176,7 +176,7 @@ public class FlowService {
|
||||
previous :
|
||||
FlowWithSource.of(flowToImport.toBuilder().revision(previous.getRevision() + 1).build(), source)
|
||||
)
|
||||
.orElseGet(() -> FlowWithSource.of(flowToImport, source).toBuilder().revision(1).build());
|
||||
.orElseGet(() -> FlowWithSource.of(flowToImport, source).toBuilder().tenantId(tenantId).revision(1).build());
|
||||
} else {
|
||||
return maybeExisting
|
||||
.map(previous -> repository().update(flow, previous))
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package io.kestra.core.services;
|
||||
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
@Singleton
|
||||
public class MaintenanceService {
|
||||
/**
|
||||
* @return true if the cluster is in maintenance mode
|
||||
*/
|
||||
public boolean isInMaintenanceMode() {
|
||||
// maintenance mode is an EE feature
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -5,16 +5,19 @@ import io.kestra.core.test.TestState;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
|
||||
public record UnitTestResult(
|
||||
@NotNull
|
||||
String unitTestId,
|
||||
String testId,
|
||||
@NotNull
|
||||
String unitTestType,
|
||||
String testType,
|
||||
@NotNull
|
||||
String executionId,
|
||||
@NotNull
|
||||
URI url,
|
||||
@NotNull
|
||||
TestState state,
|
||||
@NotNull
|
||||
List<AssertionResult> assertionResults,
|
||||
@@ -22,14 +25,13 @@ public record UnitTestResult(
|
||||
List<AssertionRunError> errors,
|
||||
Fixtures fixtures
|
||||
) {
|
||||
|
||||
public static UnitTestResult of(String unitTestId, String unitTestType, String executionId, List<AssertionResult> results, List<AssertionRunError> errors, @Nullable Fixtures fixtures) {
|
||||
public static UnitTestResult of(String unitTestId, String unitTestType, String executionId, URI url, List<AssertionResult> results, List<AssertionRunError> errors, @Nullable Fixtures fixtures) {
|
||||
TestState state;
|
||||
if(!errors.isEmpty()){
|
||||
state = TestState.ERROR;
|
||||
} else {
|
||||
state = results.stream().anyMatch(assertion -> !assertion.isSuccess()) ? TestState.FAILED : TestState.SUCCESS;
|
||||
}
|
||||
return new UnitTestResult(unitTestId, unitTestType, executionId, state, results, errors, fixtures);
|
||||
return new UnitTestResult(unitTestId, unitTestType, executionId, url, state, results, errors, fixtures);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ public class GraphUtils {
|
||||
)))
|
||||
.orElse(Collections.emptyMap());
|
||||
|
||||
triggersDeclarations.forEach(trigger -> {
|
||||
triggersDeclarations.stream().filter(trigger -> trigger != null).forEach(trigger -> {
|
||||
GraphTrigger triggerNode = new GraphTrigger(trigger, triggersById.get(trigger.getId()));
|
||||
triggerCluster.addNode(triggerNode);
|
||||
triggerCluster.addEdge(triggerCluster.getRoot(), triggerNode, new Relation());
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package io.kestra.core.utils;
|
||||
|
||||
import io.kestra.core.models.Setting;
|
||||
import io.kestra.core.repositories.SettingRepositoryInterface;
|
||||
import io.micronaut.context.env.Environment;
|
||||
import io.micronaut.context.env.PropertiesPropertySourceLoader;
|
||||
import io.micronaut.context.env.PropertySource;
|
||||
@@ -29,6 +31,9 @@ public class VersionProvider {
|
||||
@Inject
|
||||
private Environment environment;
|
||||
|
||||
@Inject
|
||||
private Optional<SettingRepositoryInterface> settingRepository; // repositories are not always there on unit tests
|
||||
|
||||
@PostConstruct
|
||||
public void start() {
|
||||
final Optional<PropertySource> gitProperties = new PropertiesPropertySourceLoader()
|
||||
@@ -40,6 +45,18 @@ public class VersionProvider {
|
||||
this.revision = loadRevision(gitProperties);
|
||||
this.date = loadTime(gitProperties);
|
||||
this.version = loadVersion(buildProperties, gitProperties);
|
||||
|
||||
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
|
||||
if (settingRepository.isPresent()) {
|
||||
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
|
||||
settingRepository.get().save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(this.version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String loadVersion(final Optional<PropertySource> buildProperties,
|
||||
|
||||
@@ -111,8 +111,9 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
})
|
||||
).collect(Collectors.toMap(
|
||||
Map.Entry::getKey,
|
||||
Map.Entry::getValue
|
||||
));
|
||||
Map.Entry::getValue,
|
||||
(first, second) -> second)
|
||||
);
|
||||
} else if (labels instanceof Map<?, ?> map) {
|
||||
labelsAsMap = map.entrySet()
|
||||
.stream()
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package io.kestra.plugin.core.execution;
|
||||
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutionUpdatableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@SuperBuilder
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Schema(
|
||||
title = "Allow to set execution variables. These variables will then be available via the `{{ vars.name }}` expression."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
full = true,
|
||||
title = "Set variables",
|
||||
code = """
|
||||
id: variables
|
||||
namespace: company.team
|
||||
|
||||
variables:
|
||||
name: World
|
||||
|
||||
tasks:
|
||||
- id: set_vars
|
||||
type: io.kestra.plugin.core.execution.SetVariables
|
||||
variables:
|
||||
message: Hello
|
||||
name: Loïc
|
||||
- id: hello
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: "{{ vars.message }} {{ vars.name }}\""""
|
||||
)
|
||||
}
|
||||
)
|
||||
public class SetVariables extends Task implements ExecutionUpdatableTask {
|
||||
@Schema(title = "The variables")
|
||||
@NotNull
|
||||
private Property<Map<String, Object>> variables;
|
||||
|
||||
@Schema(title = "Whether to overwrite existing variables")
|
||||
@NotNull
|
||||
@Builder.Default
|
||||
private Property<Boolean> overwrite = Property.ofValue(true);
|
||||
|
||||
@Override
|
||||
public Execution update(Execution execution, RunContext runContext) throws Exception {
|
||||
Map<String, Object> renderedVars = runContext.render(this.variables).asMap(String.class, Object.class);
|
||||
boolean renderedOverwrite = runContext.render(overwrite).as(Boolean.class).orElseThrow();
|
||||
if (!renderedOverwrite) {
|
||||
// check that none of the new variables already exist
|
||||
List<String> duplicated = renderedVars.keySet().stream().filter(key -> execution.getVariables().containsKey(key)).toList();
|
||||
if (!duplicated.isEmpty()) {
|
||||
throw new IllegalArgumentException("`overwrite` is set to false and the following variables already exist: " + String.join(",", duplicated));
|
||||
}
|
||||
}
|
||||
|
||||
return execution.withVariables(MapUtils.merge(execution.getVariables(), renderedVars));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package io.kestra.plugin.core.execution;
|
||||
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutionUpdatableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@SuperBuilder
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Schema(
|
||||
title = "Allow to unset execution variables."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
full = true,
|
||||
title = "Set and later unset variables",
|
||||
code = """
|
||||
id: variables
|
||||
namespace: company.team
|
||||
|
||||
variables:
|
||||
name: World
|
||||
|
||||
tasks:
|
||||
- id: set_vars
|
||||
type: io.kestra.plugin.core.execution.SetVariables
|
||||
variables:
|
||||
message: Hello
|
||||
name: Loïc
|
||||
- id: hello
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: "{{ vars.message }} {{ vars.name }}"
|
||||
- id: unset_variables
|
||||
type: io.kestra.plugin.core.execution.UnsetVariables
|
||||
variables:
|
||||
- message
|
||||
- name"""
|
||||
)
|
||||
}
|
||||
)
|
||||
public class UnsetVariables extends Task implements ExecutionUpdatableTask {
|
||||
@Schema(title = "The variables")
|
||||
@NotNull
|
||||
private Property<List<String>> variables;
|
||||
|
||||
@Schema(title = "Whether to ignore missing variables")
|
||||
@NotNull
|
||||
@Builder.Default
|
||||
private Property<Boolean> ignoreMissing = Property.ofValue(false);
|
||||
|
||||
|
||||
@Override
|
||||
public Execution update(Execution execution, RunContext runContext) throws Exception {
|
||||
List<String> renderedVariables = runContext.render(variables).asList(String.class);
|
||||
boolean renderedIgnoreMissing = runContext.render(ignoreMissing).as(Boolean.class).orElseThrow();
|
||||
Map<String, Object> variables = execution.getVariables();
|
||||
for (String key : renderedVariables) {
|
||||
removeVar(variables, key, renderedIgnoreMissing);
|
||||
}
|
||||
return execution.withVariables(variables);
|
||||
}
|
||||
|
||||
private void removeVar(Map<String, Object> vars, String key, boolean ignoreMissing) {
|
||||
if (key.indexOf('.') >= 0) {
|
||||
String prefix = key.substring(0, key.indexOf('.'));
|
||||
String suffix = key.substring(key.indexOf('.') + 1);
|
||||
removeVar((Map<String, Object>) vars.get(prefix), suffix, ignoreMissing);
|
||||
} else {
|
||||
if (ignoreMissing && !vars.containsKey(key)) {
|
||||
return;
|
||||
}
|
||||
vars.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -555,7 +555,7 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
builder.uri(uri);
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
|
||||
var state = this.isAllowFailure() ? State.Type.WARNING : State.Type.FAILED;
|
||||
var state = State.Type.fail(this);
|
||||
taskRun = taskRun
|
||||
.withState(state)
|
||||
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
|
||||
|
||||
@@ -29,8 +29,13 @@ import java.util.concurrent.TimeUnit;
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
full = true,
|
||||
code = """
|
||||
id: sleep
|
||||
id: sleep
|
||||
namespace: company.team
|
||||
|
||||
tasks:
|
||||
- id: sleep
|
||||
type: io.kestra.plugin.core.flow.Sleep
|
||||
duration: "PT5S"
|
||||
"""
|
||||
|
||||
@@ -208,48 +208,50 @@ public class Subflow extends Task implements ExecutableTask<Subflow.Output>, Chi
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
boolean isOutputsAllowed = runContext
|
||||
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
|
||||
.orElse(true);
|
||||
|
||||
final Output.OutputBuilder builder = Output.builder()
|
||||
.executionId(execution.getId())
|
||||
.state(execution.getState().getCurrent());
|
||||
|
||||
final Map<String, Object> subflowOutputs = Optional
|
||||
.ofNullable(flow.getOutputs())
|
||||
.map(outputs -> outputs
|
||||
.stream()
|
||||
.collect(Collectors.toMap(
|
||||
io.kestra.core.models.flows.Output::getId,
|
||||
io.kestra.core.models.flows.Output::getValue)
|
||||
)
|
||||
)
|
||||
.orElseGet(() -> isOutputsAllowed ? this.getOutputs() : null);
|
||||
|
||||
VariablesService variablesService = ((DefaultRunContext) runContext).getApplicationContext().getBean(VariablesService.class);
|
||||
if (subflowOutputs != null) {
|
||||
try {
|
||||
Map<String, Object> outputs = runContext.render(subflowOutputs);
|
||||
FlowInputOutput flowInputOutput = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowInputOutput.class); // this is hacking
|
||||
if (flow.getOutputs() != null && flowInputOutput != null) {
|
||||
outputs = flowInputOutput.typedOutputs(flow, execution, outputs);
|
||||
}
|
||||
builder.outputs(outputs);
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
|
||||
var state = this.isAllowFailure() ? this.isAllowWarning() ? State.Type.SUCCESS : State.Type.WARNING : State.Type.FAILED;
|
||||
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
|
||||
taskRun = taskRun
|
||||
.withState(state)
|
||||
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
|
||||
.withOutputs(variables);
|
||||
if (this.wait) { // we only compute outputs if we wait for the subflow
|
||||
boolean isOutputsAllowed = runContext
|
||||
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
|
||||
.orElse(true);
|
||||
|
||||
return Optional.of(SubflowExecutionResult.builder()
|
||||
.executionId(execution.getId())
|
||||
.state(State.Type.FAILED)
|
||||
.parentTaskRun(taskRun)
|
||||
.build());
|
||||
final Map<String, Object> subflowOutputs = Optional
|
||||
.ofNullable(flow.getOutputs())
|
||||
.map(outputs -> outputs
|
||||
.stream()
|
||||
.collect(Collectors.toMap(
|
||||
io.kestra.core.models.flows.Output::getId,
|
||||
io.kestra.core.models.flows.Output::getValue)
|
||||
)
|
||||
)
|
||||
.orElseGet(() -> isOutputsAllowed ? this.getOutputs() : null);
|
||||
|
||||
if (subflowOutputs != null) {
|
||||
try {
|
||||
Map<String, Object> outputs = runContext.render(subflowOutputs);
|
||||
FlowInputOutput flowInputOutput = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowInputOutput.class); // this is hacking
|
||||
if (flow.getOutputs() != null && flowInputOutput != null) {
|
||||
outputs = flowInputOutput.typedOutputs(flow, execution, outputs);
|
||||
}
|
||||
builder.outputs(outputs);
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
|
||||
var state = State.Type.fail(this);
|
||||
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
|
||||
taskRun = taskRun
|
||||
.withState(state)
|
||||
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
|
||||
.withOutputs(variables);
|
||||
|
||||
return Optional.of(SubflowExecutionResult.builder()
|
||||
.executionId(execution.getId())
|
||||
.state(State.Type.FAILED)
|
||||
.parentTaskRun(taskRun)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -82,12 +82,12 @@ import java.util.stream.Stream;
|
||||
code = """
|
||||
id: daily_flow
|
||||
namespace: company.team
|
||||
|
||||
|
||||
tasks:
|
||||
- id: log
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: It's {{ trigger.date ?? taskrun.startDate | date("HH:mm") }}
|
||||
|
||||
|
||||
triggers:
|
||||
- id: schedule
|
||||
type: io.kestra.plugin.core.trigger.Schedule
|
||||
@@ -437,13 +437,6 @@ public class Schedule extends AbstractTrigger implements Schedulable, TriggerOut
|
||||
Optional.empty()
|
||||
);
|
||||
|
||||
execution = execution.toBuilder()
|
||||
// keep to avoid breaking compatibility
|
||||
.variables(ImmutableMap.of(
|
||||
"schedule", execution.getTrigger().getVariables()
|
||||
))
|
||||
.build();
|
||||
|
||||
return Optional.of(execution);
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ class DocumentationGeneratorTest {
|
||||
assertThat(render).contains("description: \"Short description for this task\"");
|
||||
assertThat(render).contains("`VALUE_1`");
|
||||
assertThat(render).contains("`VALUE_2`");
|
||||
assertThat(render).contains("This plugin is exclusively available on the Cloud and Enterprise editions of Kestra.");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.kestra.core.http.client;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.google.common.net.HttpHeaders;
|
||||
import io.kestra.core.context.TestRunContextFactory;
|
||||
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
@@ -262,6 +263,30 @@ class HttpClientTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postCustomObject_WithUnknownResponseField() throws IllegalVariableEvaluationException, HttpClientException, IOException {
|
||||
CustomObject test = CustomObject.builder()
|
||||
.id(IdUtils.create())
|
||||
.name("test")
|
||||
.build();
|
||||
|
||||
Map<String, String> withAdditionalField = JacksonMapper.ofJson().convertValue(test, new TypeReference<>() {
|
||||
});
|
||||
|
||||
withAdditionalField.put("foo", "bar");
|
||||
|
||||
try (HttpClient client = client()) {
|
||||
HttpResponse<CustomObject> response = client.request(
|
||||
HttpRequest.of(URI.create(embeddedServerUri + "/http/json-post"), "POST", HttpRequest.JsonRequestBody.builder().content(withAdditionalField).build()),
|
||||
CustomObject.class
|
||||
);
|
||||
|
||||
assertThat(response.getStatus().getCode()).isEqualTo(200);
|
||||
assertThat(response.getBody().id).isEqualTo(test.id);
|
||||
assertThat(response.getHeaders().firstValue(HttpHeaders.CONTENT_TYPE).orElseThrow()).isEqualTo(MediaType.APPLICATION_JSON);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void postMultipart() throws IOException, URISyntaxException, IllegalVariableEvaluationException, HttpClientException {
|
||||
Map<String, Object> multipart = Map.of(
|
||||
@@ -509,4 +534,4 @@ class HttpClientTest {
|
||||
String id;
|
||||
String name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class LabelTest {
|
||||
|
||||
@Test
|
||||
@@ -15,9 +16,8 @@ class LabelTest {
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
);
|
||||
|
||||
Assertions.assertEquals(
|
||||
Map.of("system", Map.of("username", "test", "correlationId", "id")),
|
||||
result
|
||||
assertThat(result).isEqualTo(
|
||||
Map.of("system", Map.of("username", "test", "correlationId", "id"))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -29,9 +29,48 @@ class LabelTest {
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
);
|
||||
|
||||
Assertions.assertEquals(
|
||||
Map.of("system", Map.of("username", "test1", "correlationId", "id")),
|
||||
result
|
||||
assertThat(result).isEqualTo(
|
||||
Map.of("system", Map.of("username", "test2", "correlationId", "id"))
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGetMapGivenDistinctLabels() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
new Label(Label.USERNAME, "test"),
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
);
|
||||
|
||||
assertThat(result).isEqualTo(
|
||||
Map.of(Label.USERNAME, "test", Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGetMapGivenDuplicateLabels() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
new Label(Label.USERNAME, "test1"),
|
||||
new Label(Label.USERNAME, "test2"),
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
);
|
||||
|
||||
assertThat(result).isEqualTo(
|
||||
Map.of(Label.USERNAME, "test2", Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDuplicateLabelsWithKeyOrderKept() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
new Label(Label.USERNAME, "test1"),
|
||||
new Label(Label.USERNAME, "test2"),
|
||||
new Label(Label.CORRELATION_ID, "id"),
|
||||
new Label(Label.USERNAME, "test3"))
|
||||
);
|
||||
|
||||
assertThat(result).containsExactly(
|
||||
new Label(Label.USERNAME, "test3"),
|
||||
new Label(Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package io.kestra.core.models.executions;
|
||||
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import io.kestra.core.models.flows.State;
|
||||
|
||||
@@ -157,7 +158,57 @@ class ExecutionTest {
|
||||
.labels(List.of(new Label("test", "test-value")))
|
||||
.build();
|
||||
|
||||
assertThat(execution.getLabels().size()).isEqualTo(1);
|
||||
assertThat(execution.getLabels().getFirst()).isEqualTo(new Label("test", "test-value"));
|
||||
assertThat(execution.getLabels()).containsExactly(new Label("test", "test-value"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void labelsGetDeduplicated() {
|
||||
final List<Label> duplicatedLabels = List.of(
|
||||
new Label("test", "value1"),
|
||||
new Label("test", "value2")
|
||||
);
|
||||
|
||||
final Execution executionWithLabels = Execution.builder()
|
||||
.build()
|
||||
.withLabels(duplicatedLabels);
|
||||
assertThat(executionWithLabels.getLabels()).containsExactly(new Label("test", "value2"));
|
||||
|
||||
final Execution executionBuilder = Execution.builder()
|
||||
.labels(duplicatedLabels)
|
||||
.build();
|
||||
assertThat(executionBuilder.getLabels()).containsExactly(new Label("test", "value2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Disabled("Solve label deduplication on instantization")
|
||||
void labelsGetDeduplicatedOnNewInstance() {
|
||||
final List<Label> duplicatedLabels = List.of(
|
||||
new Label("test", "value1"),
|
||||
new Label("test", "value2")
|
||||
);
|
||||
|
||||
final Execution executionNew = new Execution(
|
||||
"foo",
|
||||
"id",
|
||||
"namespace",
|
||||
"flowId",
|
||||
1,
|
||||
Collections.emptyList(),
|
||||
Map.of(),
|
||||
Map.of(),
|
||||
duplicatedLabels,
|
||||
Map.of(),
|
||||
State.of(State.Type.SUCCESS, Collections.emptyList()),
|
||||
"parentId",
|
||||
"originalId",
|
||||
null,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
);
|
||||
assertThat(executionNew.getLabels()).containsExactly(new Label("test", "value2"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ class MultiselectInputTest {
|
||||
MultiselectInput input = MultiselectInput
|
||||
.builder()
|
||||
.id("id")
|
||||
.expression("{{ values }}")
|
||||
.expression("{{ values }}\n")
|
||||
.build();
|
||||
// When
|
||||
Input<?> renderInput = RenderableInput.mayRenderInput(input, s -> {
|
||||
@@ -60,4 +60,4 @@ class MultiselectInputTest {
|
||||
// Then
|
||||
Assertions.assertEquals(((MultiselectInput)renderInput).getValues(), List.of("1", "2"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ class SelectInputTest {
|
||||
SelectInput input = SelectInput
|
||||
.builder()
|
||||
.id("id")
|
||||
.expression("{{ values }}")
|
||||
.expression("{{ values }}\n")
|
||||
.build();
|
||||
// When
|
||||
Input<?> renderInput = RenderableInput.mayRenderInput(input, s -> {
|
||||
@@ -60,4 +60,4 @@ class SelectInputTest {
|
||||
// Then
|
||||
Assertions.assertEquals(((SelectInput)renderInput).getValues(), List.of("1", "2"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,31 @@ class ScriptServiceTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void replaceInternalStorageUnicode() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file-龍.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
|
||||
String internalStorageUri = "kestra://some/file-龍.txt";
|
||||
File localFile = null;
|
||||
try {
|
||||
var command = ScriptService.replaceInternalStorage(runContext, "my command with an internal storage file: " + internalStorageUri, false);
|
||||
|
||||
Matcher matcher = COMMAND_PATTERN_CAPTURE_LOCAL_PATH.matcher(command);
|
||||
assertThat(matcher.matches()).isTrue();
|
||||
Path absoluteLocalFilePath = Path.of(matcher.group(1));
|
||||
localFile = absoluteLocalFilePath.toFile();
|
||||
assertThat(localFile.exists()).isTrue();
|
||||
} finally {
|
||||
localFile.delete();
|
||||
path.toFile().delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void uploadInputFiles() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.fasterxml.jackson.databind.node.TextNode;
|
||||
import io.kestra.core.models.Plugin;
|
||||
import io.kestra.core.plugins.PluginRegistry;
|
||||
@@ -15,12 +16,14 @@ import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PluginDeserializerTest {
|
||||
|
||||
@Mock
|
||||
private PluginRegistry registry;
|
||||
|
||||
|
||||
@Test
|
||||
void shouldSucceededDeserializePluginGivenValidType() throws JsonProcessingException {
|
||||
// Given
|
||||
@@ -38,8 +41,9 @@ class PluginDeserializerTest {
|
||||
|
||||
TestPluginHolder deserialized = om.readValue(input, TestPluginHolder.class);
|
||||
// Then
|
||||
Assertions.assertEquals(TestPlugin.class.getCanonicalName(), deserialized.plugin().getType());
|
||||
Mockito.verify(registry, Mockito.only()).findClassByIdentifier(identifier);
|
||||
assertThat(TestPlugin.class.getCanonicalName()).isEqualTo(deserialized.plugin().getType());
|
||||
Mockito.verify(registry, Mockito.times(1)).isVersioningSupported();
|
||||
Mockito.verify(registry, Mockito.times(1)).findClassByIdentifier(identifier);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -57,17 +61,33 @@ class PluginDeserializerTest {
|
||||
});
|
||||
|
||||
// Then
|
||||
Assertions.assertEquals("io.kestra.core.plugins.serdes.Unknown", exception.getTypeId());
|
||||
assertThat("io.kestra.core.plugins.serdes.Unknown").isEqualTo(exception.getTypeId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnNullPluginIdentifierGivenNullType() {
|
||||
Assertions.assertNull(PluginDeserializer.extractPluginRawIdentifier(new TextNode(null)));
|
||||
assertThat(PluginDeserializer.extractPluginRawIdentifier(new TextNode(null), true)).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnNullPluginIdentifierGivenEmptyType() {
|
||||
Assertions.assertNull(PluginDeserializer.extractPluginRawIdentifier(new TextNode("")));
|
||||
assertThat(PluginDeserializer.extractPluginRawIdentifier(new TextNode(""), true)).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnTypeWithVersionGivenSupportedVersionTrue() {
|
||||
ObjectNode jsonNodes = new ObjectNode(new ObjectMapper().getNodeFactory());
|
||||
jsonNodes.set("type", new TextNode("io.kestra.core.plugins.serdes.Unknown"));
|
||||
jsonNodes.set("version", new TextNode("1.0.0"));
|
||||
assertThat(PluginDeserializer.extractPluginRawIdentifier(jsonNodes, true)).isEqualTo("io.kestra.core.plugins.serdes.Unknown:1.0.0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnTypeWithVersionGivenSupportedVersionFalse() {
|
||||
ObjectNode jsonNodes = new ObjectNode(new ObjectMapper().getNodeFactory());
|
||||
jsonNodes.set("type", new TextNode("io.kestra.core.plugins.serdes.Unknown"));
|
||||
jsonNodes.set("version", new TextNode("1.0.0"));
|
||||
assertThat(PluginDeserializer.extractPluginRawIdentifier(jsonNodes, false)).isEqualTo("io.kestra.core.plugins.serdes.Unknown");
|
||||
}
|
||||
|
||||
public record TestPluginHolder(Plugin plugin) {
|
||||
|
||||
@@ -185,6 +185,18 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartSubflow();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart-with-finally.yaml"})
|
||||
protected void restartFailedWithFinally() throws Exception {
|
||||
restartCaseTest.restartFailedWithFinally();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart-with-after-execution.yaml"})
|
||||
protected void restartFailedWithAfterExecution() throws Exception {
|
||||
restartCaseTest.restartFailedWithAfterExecution();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
"flows/valids/trigger-flow-listener.yaml",
|
||||
@@ -405,6 +417,12 @@ public abstract class AbstractRunnerTest {
|
||||
flowConcurrencyCaseTest.flowConcurrencyCancelPause();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"})
|
||||
protected void flowConcurrencyWithForEachItem() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem();
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/executable-fail.yml")
|
||||
void badExecutable(Execution execution) {
|
||||
@@ -492,6 +510,12 @@ public abstract class AbstractRunnerTest {
|
||||
slaTestCase.executionConditionSLAShouldLabel();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/sla-parent-flow.yaml", "flows/valids/sla-subflow.yaml"})
|
||||
void executionConditionSLAShouldLaslaViolationOnSubflowMayEndTheParentFlowbel() throws Exception {
|
||||
slaTestCase.slaViolationOnSubflowMayEndTheParentFlow();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/if.yaml"})
|
||||
void multipleIf() throws TimeoutException, QueueException {
|
||||
|
||||
@@ -8,18 +8,28 @@ import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -28,7 +38,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
@Singleton
|
||||
public class FlowConcurrencyCaseTest {
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private StorageInterface storageInterface;
|
||||
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
@@ -237,4 +253,49 @@ public class FlowConcurrencyCaseTest {
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
|
||||
}
|
||||
|
||||
public void flowConcurrencyWithForEachItem() throws TimeoutException, QueueException, InterruptedException, URISyntaxException, IOException {
|
||||
URI file = storageUpload();
|
||||
Map<String, Object> inputs = Map.of("file", file.toString(), "batch", 4);
|
||||
Execution forEachItem = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-for-each-item", null,
|
||||
(flow, execution1) -> flowIO.readExecutionInputs(flow, execution1, inputs), Duration.ofSeconds(5));
|
||||
assertThat(forEachItem.getState().getCurrent()).isEqualTo(Type.RUNNING);
|
||||
|
||||
Set<String> executionIds = new HashSet<>();
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if ("flow-concurrency-queue".equals(e.getLeft().getFlowId()) && e.getLeft().getState().isRunning()) {
|
||||
executionIds.add(e.getLeft().getId());
|
||||
}
|
||||
});
|
||||
|
||||
// wait a little to be sure there are not too many executions started
|
||||
Thread.sleep(500);
|
||||
|
||||
assertThat(executionIds).hasSize(1);
|
||||
receive.blockLast();
|
||||
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getId().equals(forEachItem.getId()) && e.getState().isTerminated(), () -> {}, Duration.ofSeconds(10));
|
||||
assertThat(terminated.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
}
|
||||
|
||||
private URI storageUpload() throws URISyntaxException, IOException {
|
||||
File tempFile = File.createTempFile("file", ".txt");
|
||||
|
||||
Files.write(tempFile.toPath(), content());
|
||||
|
||||
return storageInterface.put(
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
new URI("/file/storage/file.txt"),
|
||||
new FileInputStream(tempFile)
|
||||
);
|
||||
}
|
||||
|
||||
private List<String> content() {
|
||||
return IntStream
|
||||
.range(0, 7)
|
||||
.mapToObj(value -> StringUtils.leftPad(value + "", 20))
|
||||
.toList();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -261,4 +261,78 @@ public class RestartCaseTest {
|
||||
assertTrue(successLatch.await(1, TimeUnit.MINUTES));
|
||||
receiveSubflows.blockLast();
|
||||
}
|
||||
|
||||
public void restartFailedWithFinally() throws Exception {
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "restart-with-finally").orElseThrow();
|
||||
|
||||
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId(), Duration.ofSeconds(60));
|
||||
|
||||
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(firstExecution.getTaskRunList()).hasSize(3);
|
||||
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.SUCCESS && execution.getId().equals(firstExecution.getId()),
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
executionQueue.emit(restartedExec);
|
||||
}),
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
assertThat(finishedRestartedExecution.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(finishedRestartedExecution.getParentId()).isNull();
|
||||
assertThat(finishedRestartedExecution.getTaskRunList().size()).isEqualTo(4);
|
||||
|
||||
finishedRestartedExecution
|
||||
.getTaskRunList()
|
||||
.stream()
|
||||
.map(TaskRun::getState)
|
||||
.forEach(state -> assertThat(state.getCurrent()).isIn(State.Type.SUCCESS, State.Type.SKIPPED));
|
||||
}
|
||||
|
||||
public void restartFailedWithAfterExecution() throws Exception {
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "restart-with-after-execution").orElseThrow();
|
||||
|
||||
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId(), Duration.ofSeconds(60));
|
||||
|
||||
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(firstExecution.getTaskRunList()).hasSize(3);
|
||||
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess() && execution.getId().equals(firstExecution.getId()),
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
executionQueue.emit(restartedExec);
|
||||
}),
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
assertThat(finishedRestartedExecution.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(finishedRestartedExecution.getParentId()).isNull();
|
||||
assertThat(finishedRestartedExecution.getTaskRunList().size()).isEqualTo(4);
|
||||
|
||||
finishedRestartedExecution
|
||||
.getTaskRunList()
|
||||
.stream()
|
||||
.map(TaskRun::getState)
|
||||
.forEach(state -> assertThat(state.getCurrent()).isIn(State.Type.SUCCESS, State.Type.SKIPPED));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,4 +48,10 @@ public class SLATestCase {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(execution.getLabels()).contains(new Label("sla", "violated"));
|
||||
}
|
||||
|
||||
public void slaViolationOnSubflowMayEndTheParentFlow() throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-parent-flow");
|
||||
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package io.kestra.core.runners.pebble.filters;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.runners.VariableRenderer;
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import jakarta.inject.Inject;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
@KestraTest
|
||||
public class DateAddFilterTest {
|
||||
|
||||
@Inject
|
||||
VariableRenderer variableRenderer;
|
||||
|
||||
@Test
|
||||
void toDateAdd() throws IllegalVariableEvaluationException {
|
||||
Map<String, Object> vars = Map.of(
|
||||
"day", 1,
|
||||
"timezone", "Australia/Perth"
|
||||
);
|
||||
String render = variableRenderer.render(
|
||||
"{{ \"2013-09-08T16:19:00+02\" | dateAdd(day, \"DAYS\") | date(\"yyyy-MM-dd HH:mm:ss z\", timeZone=render(timezone)) }}",
|
||||
vars);
|
||||
|
||||
assertThat(render).isEqualTo("2013-09-09 22:19:00 AWST");
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_fail_with_invalid_day() {
|
||||
Map<String, Object> vars = Map.of(
|
||||
"day", "invalid",
|
||||
"timezone", "Australia/Perth"
|
||||
);
|
||||
assertThrows(IllegalVariableEvaluationException.class, () ->variableRenderer.render(
|
||||
"{{ \"2013-09-08T16:19:00+02\" | dateAdd(day, \"DAYS\") | date(\"yyyy-MM-dd HH:mm:ss z\", timeZone=render(timezone)) }}",
|
||||
vars));
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_return_null_for_null_input() throws IllegalVariableEvaluationException {
|
||||
Map<String, Object> vars = Map.of(
|
||||
"day", 1,
|
||||
"timezone", "Australia/Perth"
|
||||
);
|
||||
String render = variableRenderer.render(
|
||||
"{{ null | dateAdd(day, \"DAYS\") | date(\"yyyy-MM-dd HH:mm:ss z\", timeZone=render(timezone)) }}",
|
||||
vars);
|
||||
assertThat(render).isEmpty();
|
||||
}
|
||||
|
||||
@MethodSource("longInput")
|
||||
@ParameterizedTest
|
||||
void should_get_as_long(InputWrapper wrapper){
|
||||
assertThat(DateAddFilter.getAsLong(wrapper.value, 0, null)).isEqualTo(1L);
|
||||
}
|
||||
|
||||
static Stream<InputWrapper> longInput(){
|
||||
return Stream.of(
|
||||
new InputWrapper(1L),
|
||||
new InputWrapper(1),
|
||||
new InputWrapper(new AtomicInteger(1)),
|
||||
new InputWrapper("1")
|
||||
);
|
||||
}
|
||||
|
||||
@MethodSource("invalidInput")
|
||||
@ParameterizedTest
|
||||
void should_get_not_get_as_long(InputWrapper wrapper){
|
||||
assertThrows(PebbleException.class, () -> DateAddFilter.getAsLong(wrapper.value, 0, null));
|
||||
}
|
||||
|
||||
static Stream<InputWrapper> invalidInput(){
|
||||
return Stream.of(
|
||||
new InputWrapper(null),
|
||||
new InputWrapper("invalidString")
|
||||
);
|
||||
}
|
||||
|
||||
//Parametrized test doesn't like Object as method parameter
|
||||
record InputWrapper(Object value) {}
|
||||
}
|
||||
@@ -165,7 +165,7 @@ class YamlParserTest {
|
||||
Flow flow = this.parse("flows/valids/minimal.yaml");
|
||||
|
||||
String s = MAPPER.writeValueAsString(flow);
|
||||
assertThat(s).isEqualTo("{\"id\":\"minimal\",\"namespace\":\"io.kestra.tests\",\"revision\":2,\"disabled\":false,\"deleted\":false,\"labels\":[{\"key\":\"system.readOnly\",\"value\":\"true\"}],\"tasks\":[{\"id\":\"date\",\"type\":\"io.kestra.plugin.core.debug.Return\",\"format\":\"{{taskrun.startDate}}\"}]}");
|
||||
assertThat(s).isEqualTo("{\"id\":\"minimal\",\"namespace\":\"io.kestra.tests\",\"revision\":2,\"disabled\":false,\"deleted\":false,\"labels\":[{\"key\":\"system.readOnly\",\"value\":\"true\"},{\"key\":\"existing\",\"value\":\"label\"}],\"tasks\":[{\"id\":\"date\",\"type\":\"io.kestra.plugin.core.debug.Return\",\"format\":\"{{taskrun.startDate}}\"}]}");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package io.kestra.plugin.core.execution;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
class SetVariablesTest {
|
||||
|
||||
@ExecuteFlow("flows/valids/set-variables.yaml")
|
||||
@Test
|
||||
void shouldUpdateExecution(Execution execution) {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(execution.getTaskRunList()).hasSize(2);
|
||||
assertThat(((Map<String, Object>) execution.getTaskRunList().get(1).getOutputs().get("values"))).containsEntry("message", "Hello Loïc");
|
||||
}
|
||||
|
||||
@ExecuteFlow("flows/valids/set-variables-duplicate.yaml")
|
||||
@Test
|
||||
void shouldFailWhenExistingVariable(Execution execution) {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package io.kestra.plugin.core.execution;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
class UnsetVariablesTest {
|
||||
|
||||
@ExecuteFlow("flows/valids/unset-variables.yaml")
|
||||
@Test
|
||||
void shouldUpdateExecution(Execution execution) {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(execution.getTaskRunList()).hasSize(3);
|
||||
assertThat(((Map<String, Object>) execution.getTaskRunList().get(2).getOutputs().get("values"))).containsEntry("message", "default");
|
||||
}
|
||||
}
|
||||
@@ -54,4 +54,10 @@ class ForEachTest {
|
||||
assertThat(execution.findTaskRunsByTaskId("e1").getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(execution.findTaskRunsByTaskId("e2").getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/foreach-nested.yaml")
|
||||
void nested(Execution execution) {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,12 @@ public class PauseTest {
|
||||
suite.runTimeout(runnerUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/pause-timeout-allow-failure.yaml"})
|
||||
void timeoutAllowFailure() throws Exception {
|
||||
suite.runTimeoutAllowFailure(runnerUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/pause_no_tasks.yaml"})
|
||||
void runEmptyTasks() throws Exception {
|
||||
@@ -235,6 +241,25 @@ public class PauseTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
}
|
||||
|
||||
public void runTimeoutAllowFailure(RunnerUtils runnerUtils) throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause-timeout-allow-failure", null, null, Duration.ofSeconds(30));
|
||||
String executionId = execution.getId();
|
||||
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.PAUSED);
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
|
||||
execution = runnerUtils.awaitExecution(
|
||||
e -> e.getId().equals(executionId) && e.getState().getCurrent() == State.Type.WARNING,
|
||||
() -> {},
|
||||
Duration.ofSeconds(5)
|
||||
);
|
||||
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getHistories().stream().filter(history -> history.getState() == State.Type.PAUSED).count()).as("Task runs were: " + execution.getTaskRunList().toString()).isEqualTo(1L);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getHistories().stream().filter(history -> history.getState() == State.Type.RUNNING).count()).isEqualTo(1L);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getHistories().stream().filter(history -> history.getState() == State.Type.WARNING).count()).isEqualTo(1L);
|
||||
assertThat(execution.getTaskRunList()).hasSize(2);
|
||||
}
|
||||
|
||||
public void runEmptyTasks(RunnerUtils runnerUtils) throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause_no_tasks", null, null, Duration.ofSeconds(30));
|
||||
String executionId = execution.getId();
|
||||
|
||||
@@ -2,7 +2,6 @@ package io.kestra.plugin.core.flow;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
@@ -104,4 +103,61 @@ class RuntimeLabelsTest {
|
||||
new Label("taskRunId", labelsTaskRunId),
|
||||
new Label("existingLabel", "someValue"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/primitive-labels-flow.yml"})
|
||||
void primitiveTypeLabelsOverrideExistingLabels() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"primitive-labels-flow",
|
||||
null,
|
||||
(flow, createdExecution) -> Map.of(
|
||||
"intLabel", 42,
|
||||
"boolLabel", true,
|
||||
"floatLabel", 3.14f
|
||||
),
|
||||
null,
|
||||
List.of(
|
||||
new Label("intValue", "1"),
|
||||
new Label("boolValue", "false"),
|
||||
new Label("floatValue", "4.2f")
|
||||
)
|
||||
);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
String labelsTaskRunId = execution.findTaskRunsByTaskId("update-labels").getFirst().getId();
|
||||
|
||||
assertThat(execution.getLabels()).containsExactlyInAnyOrder(
|
||||
new Label(Label.CORRELATION_ID, execution.getId()),
|
||||
new Label("intValue", "42"),
|
||||
new Label("boolValue", "true"),
|
||||
new Label("floatValue", "3.14"),
|
||||
new Label("taskRunId", labelsTaskRunId));
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/labels-update-task-deduplicate.yml"})
|
||||
void updateGetsDeduplicated() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"labels-update-task-deduplicate",
|
||||
null,
|
||||
(flow, createdExecution) -> Map.of(),
|
||||
null,
|
||||
List.of()
|
||||
);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(2);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(execution.getLabels()).containsExactlyInAnyOrder(
|
||||
new Label(Label.CORRELATION_ID, execution.getId()),
|
||||
new Label("fromStringKey", "value2"),
|
||||
new Label("fromListKey", "value2")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package io.kestra.plugin.core.flow;
|
||||
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface;
|
||||
import io.kestra.core.runners.RunnerUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
class SubflowRunnerTest {
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/subflow-inherited-labels-child.yaml", "flows/valids/subflow-inherited-labels-parent.yaml"})
|
||||
void inheritedLabelsAreOverridden() throws QueueException, TimeoutException {
|
||||
Execution parentExecution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-inherited-labels-parent");
|
||||
|
||||
assertThat(parentExecution.getLabels()).containsExactlyInAnyOrder(
|
||||
new Label(Label.CORRELATION_ID, parentExecution.getId()),
|
||||
new Label("parentFlowLabel1", "value1"),
|
||||
new Label("parentFlowLabel2", "value2")
|
||||
);
|
||||
|
||||
String childExecutionId = (String) parentExecution.findTaskRunsByTaskId("launch").getFirst().getOutputs().get("executionId");
|
||||
|
||||
assertThat(childExecutionId).isNotBlank();
|
||||
|
||||
Execution childExecution = executionRepository.findById(MAIN_TENANT, childExecutionId).orElseThrow();
|
||||
|
||||
assertThat(childExecution.getLabels()).containsExactlyInAnyOrder(
|
||||
new Label(Label.CORRELATION_ID, parentExecution.getId()), // parent's correlation ID
|
||||
new Label("childFlowLabel1", "value1"), // defined by the subtask flow
|
||||
new Label("childFlowLabel2", "value2"), // defined by the subtask flow
|
||||
new Label("launchTaskLabel", "launchFoo"), // added by Subtask
|
||||
new Label("parentFlowLabel1", "launchBar"), // overridden by Subtask
|
||||
new Label("parentFlowLabel2", "value2") // inherited from the parent flow
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/subflow-parent-no-wait.yaml", "flows/valids/subflow-child-with-output.yaml"})
|
||||
void subflowOutputWithoutWait() throws QueueException, TimeoutException, InterruptedException {
|
||||
AtomicReference<Execution> childExecution = new AtomicReference<>();
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
Runnable closing = executionQueue.receive(either -> {
|
||||
if (either.isLeft() && either.getLeft().getFlowId().equals("subflow-child-with-output") && either.getLeft().getState().isTerminated()) {
|
||||
childExecution.set(either.getLeft());
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Execution parentExecution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-parent-no-wait");
|
||||
String childExecutionId = (String) parentExecution.findTaskRunsByTaskId("subflow").getFirst().getOutputs().get("executionId");
|
||||
assertThat(childExecutionId).isNotBlank();
|
||||
assertThat(parentExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(parentExecution.getTaskRunList()).hasSize(1);
|
||||
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
assertThat(childExecution.get().getId()).isEqualTo(childExecutionId);
|
||||
assertThat(childExecution.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(childExecution.get().getTaskRunList()).hasSize(1);
|
||||
closing.run();
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,12 @@ public class WorkingDirectoryTest {
|
||||
suite.encryption(runnerUtils, runContextFactory);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/working-directory-invalid-runif.yaml"})
|
||||
void invalidRunIf() throws Exception {
|
||||
suite.invalidRunIf(runnerUtils);
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class Suite {
|
||||
@Inject
|
||||
@@ -310,6 +316,15 @@ public class WorkingDirectoryTest {
|
||||
assertThat(execution.findTaskRunsByTaskId("decrypted").getFirst().getOutputs().get("value")).isEqualTo("Hello World");
|
||||
}
|
||||
|
||||
public void invalidRunIf(RunnerUtils runnerUtils) throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "working-directory-invalid-runif", null,
|
||||
(f, e) -> ImmutableMap.of("failed", "false"), Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(2);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
}
|
||||
|
||||
private void put(String path, String content) throws IOException {
|
||||
put(path, content, "io.kestra.tests");
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ class ScheduleTest {
|
||||
|
||||
@Test
|
||||
void failed() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("1 1 1 1 1").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("1 1 1 1 1").build();
|
||||
|
||||
Optional<Execution> evaluate = trigger.evaluate(
|
||||
conditionContext(trigger),
|
||||
@@ -82,9 +82,8 @@ class ScheduleTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void success() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("0 0 1 * *").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("0 0 1 * *").build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.withDayOfMonth(1)
|
||||
@@ -103,12 +102,12 @@ class ScheduleTest {
|
||||
assertThat(evaluate.get().getLabels()).hasSize(3);
|
||||
assertTrue(evaluate.get().getLabels().stream().anyMatch(label -> label.key().equals(Label.CORRELATION_ID)));
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
var vars = evaluate.get().getTrigger().getVariables();
|
||||
var inputs = evaluate.get().getInputs();
|
||||
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars(vars.get("next"), date)).isEqualTo(date.plusMonths(1));
|
||||
assertThat(dateFromVars(vars.get("previous"), date)).isEqualTo(date.minusMonths(1));
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars((String) vars.get("next"), date)).isEqualTo(date.plusMonths(1));
|
||||
assertThat(dateFromVars((String) vars.get("previous"), date)).isEqualTo(date.minusMonths(1));
|
||||
assertThat(evaluate.get().getLabels()).contains(new Label("flow-label-1", "flow-label-1"));
|
||||
assertThat(evaluate.get().getLabels()).contains(new Label("flow-label-2", "flow-label-2"));
|
||||
assertThat(inputs.size()).isEqualTo(2);
|
||||
@@ -118,7 +117,7 @@ class ScheduleTest {
|
||||
|
||||
@Test
|
||||
void successWithInput() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("0 0 1 * *").inputs(Map.of("input1", "input1")).build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("0 0 1 * *").inputs(Map.of("input1", "input1")).build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.withDayOfMonth(1)
|
||||
@@ -147,7 +146,7 @@ class ScheduleTest {
|
||||
@Test
|
||||
void success_withLabels() throws Exception {
|
||||
var scheduleTrigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.cron("0 0 1 * *")
|
||||
.labels(List.of(
|
||||
new Label("trigger-label-1", "trigger-label-1"),
|
||||
@@ -173,10 +172,9 @@ class ScheduleTest {
|
||||
assertThat(evaluate.get().getLabels()).contains(new Label("trigger-label-3", ""));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void everyMinute() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("* * * * *").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("* * * * *").build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.minus(Duration.ofMinutes(1))
|
||||
@@ -191,18 +189,16 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
|
||||
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars(vars.get("next"), date)).isEqualTo(date.plus(Duration.ofMinutes(1)));
|
||||
assertThat(dateFromVars(vars.get("previous"), date)).isEqualTo(date.minus(Duration.ofMinutes(1)));
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars((String) vars.get("next"), date)).isEqualTo(date.plus(Duration.ofMinutes(1)));
|
||||
assertThat(dateFromVars((String) vars.get("previous"), date)).isEqualTo(date.minus(Duration.ofMinutes(1)));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void everySecond() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("* * * * * *").withSeconds(true).build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("* * * * * *").withSeconds(true).build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.truncatedTo(ChronoUnit.SECONDS)
|
||||
@@ -215,18 +211,17 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
|
||||
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars(vars.get("next"), date)).isEqualTo(date.plus(Duration.ofSeconds(1)));
|
||||
assertThat(dateFromVars(vars.get("previous"), date)).isEqualTo(date.minus(Duration.ofSeconds(1)));
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars((String) vars.get("next"), date)).isEqualTo(date.plus(Duration.ofSeconds(1)));
|
||||
assertThat(dateFromVars((String) vars.get("previous"), date)).isEqualTo(date.minus(Duration.ofSeconds(1)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldNotReturnExecutionForBackFillWhenCurrentDateIsBeforeScheduleDate() throws Exception {
|
||||
// Given
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron(TEST_CRON_EVERYDAY_AT_8).build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron(TEST_CRON_EVERYDAY_AT_8).build();
|
||||
ZonedDateTime now = ZonedDateTime.now();
|
||||
TriggerContext triggerContext = triggerContext(now, trigger).toBuilder()
|
||||
.backfill(Backfill
|
||||
@@ -246,7 +241,7 @@ class ScheduleTest {
|
||||
void
|
||||
shouldReturnExecutionForBackFillWhenCurrentDateIsAfterScheduleDate() throws Exception {
|
||||
// Given
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron(TEST_CRON_EVERYDAY_AT_8).build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron(TEST_CRON_EVERYDAY_AT_8).build();
|
||||
ZonedDateTime now = ZonedDateTime.of(2025, 1, 1, 0, 0, 0, 0, ZoneId.systemDefault());
|
||||
TriggerContext triggerContext = triggerContext(ZonedDateTime.now(), trigger).toBuilder()
|
||||
.backfill(Backfill
|
||||
@@ -265,7 +260,7 @@ class ScheduleTest {
|
||||
|
||||
@Test
|
||||
void noBackfillNextDate() {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("0 0 * * *").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("0 0 * * *").build();
|
||||
ZonedDateTime next = trigger.nextEvaluationDate(conditionContext(trigger), Optional.empty());
|
||||
|
||||
assertThat(next.getDayOfMonth()).isEqualTo(ZonedDateTime.now().plusDays(1).getDayOfMonth());
|
||||
@@ -273,7 +268,7 @@ class ScheduleTest {
|
||||
|
||||
@Test
|
||||
void noBackfillNextDateContext() {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("0 0 * * *").timezone("Europe/Paris").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("0 0 * * *").timezone("Europe/Paris").build();
|
||||
ZonedDateTime date = ZonedDateTime.parse("2020-01-01T00:00:00+01:00[Europe/Paris]");
|
||||
ZonedDateTime next = trigger.nextEvaluationDate(conditionContext(trigger), Optional.of(triggerContext(date, trigger)));
|
||||
|
||||
@@ -281,9 +276,8 @@ class ScheduleTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
void systemBackfillChangedFromCronExpression() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("30 0 1 * *").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("30 0 1 * *").build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.withDayOfMonth(1)
|
||||
@@ -303,17 +297,16 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
assertThat(dateFromVars(vars.get("date"), expexted)).isEqualTo(expexted);
|
||||
assertThat(dateFromVars(vars.get("next"), expexted)).isEqualTo(expexted.plusMonths(1));
|
||||
assertThat(dateFromVars(vars.get("previous"), expexted)).isEqualTo(expexted.minusMonths(1));
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
assertThat(dateFromVars((String) vars.get("date"), expexted)).isEqualTo(expexted);
|
||||
assertThat(dateFromVars((String) vars.get("next"), expexted)).isEqualTo(expexted.plusMonths(1));
|
||||
assertThat(dateFromVars((String) vars.get("previous"), expexted)).isEqualTo(expexted.minusMonths(1));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void conditions() throws Exception {
|
||||
Schedule trigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.type(Schedule.class.getName())
|
||||
.cron("0 12 * * 1")
|
||||
.timezone("Europe/Paris")
|
||||
@@ -338,17 +331,16 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars(vars.get("next"), next)).isEqualTo(next);
|
||||
assertThat(dateFromVars(vars.get("previous"), previous)).isEqualTo(previous);
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars((String) vars.get("next"), next)).isEqualTo(next);
|
||||
assertThat(dateFromVars((String) vars.get("previous"), previous)).isEqualTo(previous);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void impossibleNextConditions() throws Exception {
|
||||
Schedule trigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.type(Schedule.class.getName())
|
||||
.cron("0 12 * * 1")
|
||||
.timezone("Europe/Paris")
|
||||
@@ -371,16 +363,16 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars(vars.get("previous"), previous)).isEqualTo(previous);
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(dateFromVars((String) vars.get("previous"), previous)).isEqualTo(previous);
|
||||
assertThat(vars.containsKey("next")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void lateMaximumDelay() {
|
||||
Schedule trigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.cron("* * * * *")
|
||||
.lateMaximumDelay(Duration.ofMinutes(5))
|
||||
.build();
|
||||
@@ -401,17 +393,15 @@ class ScheduleTest {
|
||||
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void hourly() throws Exception {
|
||||
Schedule trigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.cron("@hourly")
|
||||
.build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now().minusHours(1).withMinute(0).withSecond(0).withNano(0);
|
||||
|
||||
|
||||
Optional<Execution> evaluate = trigger.evaluate(
|
||||
conditionContext(trigger),
|
||||
TriggerContext.builder()
|
||||
@@ -422,14 +412,13 @@ class ScheduleTest {
|
||||
);
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void timezone() throws Exception {
|
||||
Schedule trigger = Schedule.builder().id("schedule").cron("12 9 1 * *").timezone("America/New_York").build();
|
||||
Schedule trigger = Schedule.builder().id("schedule").type(Schedule.class.getName()).cron("12 9 1 * *").timezone("America/New_York").build();
|
||||
|
||||
ZonedDateTime date = ZonedDateTime.now()
|
||||
.withZoneSameLocal(ZoneId.of("America/New_York"))
|
||||
@@ -449,18 +438,18 @@ class ScheduleTest {
|
||||
|
||||
assertThat(evaluate.isPresent()).isTrue();
|
||||
|
||||
var vars = (Map<String, String>) evaluate.get().getVariables().get("schedule");
|
||||
var vars = evaluate.get().getTrigger().getVariables();;
|
||||
|
||||
assertThat(dateFromVars(vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(ZonedDateTime.parse(vars.get("date")).getZone().getId()).isEqualTo("-04:00");
|
||||
assertThat(dateFromVars(vars.get("next"), date)).isEqualTo(date.plusMonths(1));
|
||||
assertThat(dateFromVars(vars.get("previous"), date)).isEqualTo(date.minusMonths(1));
|
||||
assertThat(dateFromVars((String) vars.get("date"), date)).isEqualTo(date);
|
||||
assertThat(ZonedDateTime.parse((String) vars.get("date")).getZone().getId()).isEqualTo("-04:00");
|
||||
assertThat(dateFromVars((String) vars.get("next"), date)).isEqualTo(date.plusMonths(1));
|
||||
assertThat(dateFromVars((String) vars.get("previous"), date)).isEqualTo(date.minusMonths(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void timezone_with_backfile() throws Exception {
|
||||
Schedule trigger = Schedule.builder()
|
||||
.id("schedule")
|
||||
.id("schedule").type(Schedule.class.getName())
|
||||
.cron(TEST_CRON_EVERYDAY_AT_8)
|
||||
.timezone("America/New_York")
|
||||
.build();
|
||||
@@ -480,8 +469,6 @@ class ScheduleTest {
|
||||
assertThat(result.isPresent()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
private ConditionContext conditionContext(AbstractTrigger trigger) {
|
||||
Flow flow = Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user