mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-25 11:12:12 -05:00
Compare commits
203 Commits
v1.0.0
...
global-sta
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f5a0dcc024 | ||
|
|
5c079b8b6b | ||
|
|
343d6b4eb9 | ||
|
|
d34d547412 | ||
|
|
7a542a24e2 | ||
|
|
5b1db68752 | ||
|
|
5b07b643d3 | ||
|
|
0e059772e4 | ||
|
|
f72e294e54 | ||
|
|
98dd884149 | ||
|
|
26c4f080fd | ||
|
|
01293de91c | ||
|
|
892b69f10e | ||
|
|
6f70d4d275 | ||
|
|
b41d2e456f | ||
|
|
5ec08eda8c | ||
|
|
7ed6b883ff | ||
|
|
eb166c9321 | ||
|
|
57aad1b931 | ||
|
|
60fe5b5c76 | ||
|
|
98c69b53bb | ||
|
|
d5d38559b4 | ||
|
|
4273ddc4f6 | ||
|
|
980c573a30 | ||
|
|
27109015f9 | ||
|
|
eba7d4f375 | ||
|
|
655a1172ee | ||
|
|
6e49a85acd | ||
|
|
4515bad6bd | ||
|
|
226dbd30c9 | ||
|
|
6b0c190edc | ||
|
|
c64df40a36 | ||
|
|
8af22d1bb2 | ||
|
|
b294457953 | ||
|
|
02d9c589fb | ||
|
|
6340d1c72f | ||
|
|
f439bd53d7 | ||
|
|
e54e3d5308 | ||
|
|
d084f2cd26 | ||
|
|
015960c78e | ||
|
|
39a09ecb67 | ||
|
|
45ce878d65 | ||
|
|
3ee647b9a8 | ||
|
|
7a7cb006bf | ||
|
|
911e6d5705 | ||
|
|
bf1458dde7 | ||
|
|
bd31e0eebd | ||
|
|
de02e4dd70 | ||
|
|
ec235b91fc | ||
|
|
ff1efa9958 | ||
|
|
e43c8ce387 | ||
|
|
2bd4e82b42 | ||
|
|
e63d6d1d86 | ||
|
|
a9752e65f2 | ||
|
|
151c56f3de | ||
|
|
d562ce3e65 | ||
|
|
681386a05b | ||
|
|
51ddfaf155 | ||
|
|
caee0a293f | ||
|
|
ba92880fa3 | ||
|
|
36b27510fb | ||
|
|
da2907e096 | ||
|
|
9b40665e64 | ||
|
|
0d35b5b355 | ||
|
|
339eb79854 | ||
|
|
0ee753529b | ||
|
|
84668fdfb9 | ||
|
|
9802f046e8 | ||
|
|
848b4d6577 | ||
|
|
1159bc5eb9 | ||
|
|
9b7ef37d14 | ||
|
|
89dfd18658 | ||
|
|
13ed2252bc | ||
|
|
c73b103bb3 | ||
|
|
396a077942 | ||
|
|
68e6fa2a4c | ||
|
|
a18748b3b2 | ||
|
|
236fcff7b4 | ||
|
|
cbbd697732 | ||
|
|
6b84737651 | ||
|
|
6ee7ecbd6b | ||
|
|
504f925085 | ||
|
|
7d37d2be93 | ||
|
|
94751a3b21 | ||
|
|
ba83b91680 | ||
|
|
56f62fb89f | ||
|
|
a0efe4b1f3 | ||
|
|
9af6338ae5 | ||
|
|
d53b933bdf | ||
|
|
a35c2816c5 | ||
|
|
d88eb9974c | ||
|
|
af3d5a384a | ||
|
|
e9ad352ccf | ||
|
|
1a95b83fb7 | ||
|
|
095939ff7a | ||
|
|
094f523874 | ||
|
|
c7efb2514a | ||
|
|
887537d8c1 | ||
|
|
0630b741b9 | ||
|
|
d2b7e723e1 | ||
|
|
080ceadf37 | ||
|
|
a89d902bc2 | ||
|
|
e2ef7d412a | ||
|
|
54c667ec4b | ||
|
|
1c53758d33 | ||
|
|
d092556bc2 | ||
|
|
308106d532 | ||
|
|
8fe8f96278 | ||
|
|
a5cad6d87c | ||
|
|
199d67fbe2 | ||
|
|
558a2e3f01 | ||
|
|
e1d2c30e54 | ||
|
|
700c6de411 | ||
|
|
2b838a5012 | ||
|
|
617daa79db | ||
|
|
1791127acb | ||
|
|
7feb571fb3 | ||
|
|
a315bd0e1c | ||
|
|
e2ac1e7e98 | ||
|
|
c6f40eff52 | ||
|
|
ccd42f7a1a | ||
|
|
ef08c8ac30 | ||
|
|
7b527c85a9 | ||
|
|
d121867066 | ||
|
|
a084a9f6f0 | ||
|
|
f6fff11081 | ||
|
|
3d5015938f | ||
|
|
951c93cedb | ||
|
|
9c06b37989 | ||
|
|
a916a03fdd | ||
|
|
4e728da331 | ||
|
|
166a3932c9 | ||
|
|
0a21971bbf | ||
|
|
8c4d7c0f9e | ||
|
|
b709913071 | ||
|
|
5be401d23c | ||
|
|
bb9f4be8c2 | ||
|
|
01e8e46b77 | ||
|
|
d00f4b0768 | ||
|
|
279f59c874 | ||
|
|
d897509726 | ||
|
|
0d592342af | ||
|
|
fc690bf7cd | ||
|
|
0a1b919863 | ||
|
|
2f4e981a29 | ||
|
|
5e7739432e | ||
|
|
8aba863b8c | ||
|
|
7eaa43c50f | ||
|
|
267ff78bfe | ||
|
|
7272cfe01f | ||
|
|
91e2fdb2cc | ||
|
|
a236688be6 | ||
|
|
81763d40ae | ||
|
|
677efb6739 | ||
|
|
b35924fef1 | ||
|
|
9dd93294b6 | ||
|
|
fac6dfe9a0 | ||
|
|
3bf9764505 | ||
|
|
c35cea5d19 | ||
|
|
4d8e9479f1 | ||
|
|
3f24e8e838 | ||
|
|
7175fcb666 | ||
|
|
2ddfa13b1b | ||
|
|
ba2a5dfec8 | ||
|
|
f84441dac7 | ||
|
|
433b788e4a | ||
|
|
65c5fd6331 | ||
|
|
421ab40276 | ||
|
|
efb2779693 | ||
|
|
74d371c0ca | ||
|
|
90a7869020 | ||
|
|
d9ccb50b0f | ||
|
|
aea0b87ef8 | ||
|
|
9a144fc3fe | ||
|
|
ddd9cebc63 | ||
|
|
1bebbb9b73 | ||
|
|
8de4dc867e | ||
|
|
fc49694e76 | ||
|
|
152300abae | ||
|
|
1ff5dda4e1 | ||
|
|
84f9b8876d | ||
|
|
575955567f | ||
|
|
d6d2580b45 | ||
|
|
070e54b902 | ||
|
|
829ca4380f | ||
|
|
381c7a75ad | ||
|
|
1688c489a9 | ||
|
|
93ccbf5f9b | ||
|
|
ac1cb235e5 | ||
|
|
9d3d3642e8 | ||
|
|
3d306a885e | ||
|
|
ef193c5774 | ||
|
|
d0f46169f4 | ||
|
|
3005ab527c | ||
|
|
688e2af12b | ||
|
|
4c0a05f484 | ||
|
|
108f8fc2c7 | ||
|
|
8b81a37559 | ||
|
|
9222f97d63 | ||
|
|
43e3591417 | ||
|
|
438dc9ecf6 | ||
|
|
7292837c58 | ||
|
|
7fa93d7764 |
4
.github/workflows/auto-translate-ui-keys.yml
vendored
4
.github/workflows/auto-translate-ui-keys.yml
vendored
@@ -26,7 +26,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: "20.x"
|
||||
|
||||
|
||||
2
.github/workflows/e2e.yml
vendored
2
.github/workflows/e2e.yml
vendored
@@ -37,7 +37,7 @@ jobs:
|
||||
path: kestra
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/.github/actions/setup-build@main
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
|
||||
14
.github/workflows/gradle-release-plugins.yml
vendored
14
.github/workflows/gradle-release-plugins.yml
vendored
@@ -25,21 +25,13 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
path: actions
|
||||
ref: main
|
||||
|
||||
# Setup build
|
||||
- uses: ./actions/.github/actions/setup-build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
caches-enabled: true
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
@@ -60,7 +52,7 @@ jobs:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
@@ -73,7 +65,7 @@ jobs:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
|
||||
9
.github/workflows/gradle-release.yml
vendored
9
.github/workflows/gradle-release.yml
vendored
@@ -38,15 +38,8 @@ jobs:
|
||||
fetch-depth: 0
|
||||
path: kestra
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
path: actions
|
||||
ref: main
|
||||
|
||||
# Setup build
|
||||
- uses: ./actions/.github/actions/setup-build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
|
||||
13
.github/workflows/main.yml
vendored
13
.github/workflows/main.yml
vendored
@@ -59,8 +59,6 @@ jobs:
|
||||
needs:
|
||||
- release
|
||||
if: always()
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
steps:
|
||||
- name: Trigger EE Workflow
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
@@ -70,14 +68,9 @@ jobs:
|
||||
repository: kestra-io/kestra-ee
|
||||
event-type: "oss-updated"
|
||||
|
||||
|
||||
# Slack
|
||||
- name: Slack - Notification
|
||||
uses: Gamesight/slack-workflow-status@master
|
||||
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
name: GitHub Actions
|
||||
icon_emoji: ":github-actions:"
|
||||
channel: "C02DQ1A7JLR" # _int_git channel
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
17
.github/workflows/pull-request.yml
vendored
17
.github/workflows/pull-request.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- develop
|
||||
- releases/*
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}-pr
|
||||
@@ -60,19 +61,3 @@ jobs:
|
||||
name: E2E - Tests
|
||||
uses: ./.github/workflows/e2e.yml
|
||||
|
||||
end:
|
||||
name: End
|
||||
runs-on: ubuntu-latest
|
||||
if: always()
|
||||
needs: [frontend, backend]
|
||||
steps:
|
||||
# Slack
|
||||
- name: Slack notification
|
||||
uses: Gamesight/slack-workflow-status@master
|
||||
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
|
||||
with:
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
name: GitHub Actions
|
||||
icon_emoji: ":github-actions:"
|
||||
channel: "C02DQ1A7JLR"
|
||||
30
.github/workflows/vulnerabilities-check.yml
vendored
30
.github/workflows/vulnerabilities-check.yml
vendored
@@ -21,13 +21,6 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
path: actions
|
||||
ref: main
|
||||
|
||||
# Setup build
|
||||
- uses: ./actions/.github/actions/setup-build
|
||||
id: build
|
||||
@@ -70,15 +63,8 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
path: actions
|
||||
ref: main
|
||||
|
||||
# Setup build
|
||||
- uses: ./actions/.github/actions/setup-build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
@@ -87,7 +73,7 @@ jobs:
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.0
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:develop
|
||||
format: 'template'
|
||||
@@ -115,24 +101,16 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
path: actions
|
||||
ref: main
|
||||
|
||||
# Setup build
|
||||
- uses: ./actions/.github/actions/setup-build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
caches-enabled: true
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.0
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:latest
|
||||
format: table
|
||||
|
||||
86
.github/workflows/workflow-backend-test.yml
vendored
86
.github/workflows/workflow-backend-test.yml
vendored
@@ -20,6 +20,7 @@ permissions:
|
||||
contents: write
|
||||
checks: write
|
||||
actions: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -35,7 +36,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/.github/actions/setup-build@main
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
@@ -59,84 +60,15 @@ jobs:
|
||||
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
|
||||
./gradlew check javadoc --parallel
|
||||
|
||||
# report test
|
||||
- name: Test - Publish Test Results
|
||||
uses: dorny/test-reporter@v2
|
||||
if: always()
|
||||
with:
|
||||
name: Java Tests Report
|
||||
reporter: java-junit
|
||||
path: '**/build/test-results/test/TEST-*.xml'
|
||||
list-suites: 'failed'
|
||||
list-tests: 'failed'
|
||||
fail-on-error: 'false'
|
||||
token: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
|
||||
# Sonar
|
||||
- name: Test - Analyze with Sonar
|
||||
if: env.SONAR_TOKEN != ''
|
||||
- name: comment PR with test report
|
||||
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
shell: bash
|
||||
run: ./gradlew sonar --info
|
||||
run: npx --yes @kestra-io/kestra-devtools generateTestReportSummary --only-errors --ci $(pwd)
|
||||
|
||||
# GCP
|
||||
- name: GCP - Auth with unit test account
|
||||
id: auth
|
||||
if: always() && env.GOOGLE_SERVICE_ACCOUNT != ''
|
||||
continue-on-error: true
|
||||
uses: "google-github-actions/auth@v3"
|
||||
with:
|
||||
credentials_json: "${{ secrets.GOOGLE_SERVICE_ACCOUNT }}"
|
||||
|
||||
- name: GCP - Setup Cloud SDK
|
||||
if: env.GOOGLE_SERVICE_ACCOUNT != ''
|
||||
uses: "google-github-actions/setup-gcloud@v3"
|
||||
|
||||
# Allure check
|
||||
- uses: rlespinasse/github-slug-action@v5
|
||||
name: Allure - Generate slug variables
|
||||
|
||||
- name: Allure - Publish report
|
||||
uses: andrcuns/allure-publish-action@v2.9.0
|
||||
if: always() && env.GOOGLE_SERVICE_ACCOUNT != ''
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
JAVA_HOME: /usr/lib/jvm/default-jvm/
|
||||
with:
|
||||
storageType: gcs
|
||||
resultsGlob: "**/build/allure-results"
|
||||
bucket: internal-kestra-host
|
||||
baseUrl: "https://internal.dev.kestra.io"
|
||||
prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }}
|
||||
copyLatest: true
|
||||
ignoreMissingResults: true
|
||||
|
||||
# Jacoco
|
||||
- name: Jacoco - Copy reports
|
||||
if: env.GOOGLE_SERVICE_ACCOUNT != ''
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: |
|
||||
mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/
|
||||
mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml
|
||||
gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }}
|
||||
|
||||
# Codecov
|
||||
- name: Codecov - Upload coverage reports
|
||||
uses: codecov/codecov-action@v5
|
||||
# Report Java
|
||||
- name: Report - Java
|
||||
uses: kestra-io/actions/composite/report-java@main
|
||||
if: ${{ !cancelled() }}
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: backend
|
||||
|
||||
- name: Codecov - Upload test results
|
||||
uses: codecov/test-results-action@v1
|
||||
if: ${{ !cancelled() }}
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: backend
|
||||
secrets: ${{ toJSON(secrets) }}
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/.github/actions/setup-build@main
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
|
||||
13
.github/workflows/workflow-github-release.yml
vendored
13
.github/workflows/workflow-github-release.yml
vendored
@@ -25,15 +25,6 @@ jobs:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
|
||||
# Checkout GitHub Actions
|
||||
- name: Checkout - Actions
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
repository: kestra-io/actions
|
||||
sparse-checkout-cone-mode: true
|
||||
path: actions
|
||||
sparse-checkout: |
|
||||
.github/actions
|
||||
|
||||
# Download Exec
|
||||
# Must be done after checkout actions
|
||||
@@ -59,7 +50,7 @@ jobs:
|
||||
|
||||
# GitHub Release
|
||||
- name: Create GitHub release
|
||||
uses: ./actions/.github/actions/github-release
|
||||
uses: kestra-io/actions/composite/github-release@main
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
env:
|
||||
MAKE_LATEST: ${{ steps.is_latest.outputs.latest }}
|
||||
@@ -82,7 +73,7 @@ jobs:
|
||||
|
||||
- name: Merge Release Notes
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
uses: ./actions/.github/actions/github-release-note-merge
|
||||
uses: kestra-io/actions/composite/github-release-note-merge@main
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
|
||||
26
.github/workflows/workflow-publish-docker.yml
vendored
26
.github/workflows/workflow-publish-docker.yml
vendored
@@ -11,6 +11,14 @@ on:
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
retag-lts:
|
||||
description: 'Retag LTS Docker images'
|
||||
required: true
|
||||
type: choice
|
||||
default: "false"
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
release-tag:
|
||||
description: 'Kestra Release Tag (by default, deduced with the ref)'
|
||||
required: false
|
||||
@@ -179,6 +187,11 @@ jobs:
|
||||
run: |
|
||||
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest{0}', matrix.image.name) }}
|
||||
|
||||
- name: Retag to LTS
|
||||
if: startsWith(github.ref, 'refs/tags/v') && inputs.retag-lts == 'true'
|
||||
run: |
|
||||
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest-lts{0}', matrix.image.name) }}
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
@@ -187,14 +200,9 @@ jobs:
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
steps:
|
||||
|
||||
# Slack
|
||||
- name: Slack notification
|
||||
uses: Gamesight/slack-workflow-status@master
|
||||
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
repo_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
name: GitHub Actions
|
||||
icon_emoji: ':github-actions:'
|
||||
channel: 'C02DQ1A7JLR' # _int_git channel
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
|
||||
2
.github/workflows/workflow-publish-maven.yml
vendored
2
.github/workflows/workflow-publish-maven.yml
vendored
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
# Setup build
|
||||
- name: Setup - Build
|
||||
uses: kestra-io/actions/.github/actions/setup-build@main
|
||||
uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
jobs:
|
||||
publish:
|
||||
name: Pull Request - Delete Docker
|
||||
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
|
||||
if: github.repository == 'kestra-io/kestra' # prevent running on forks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: dataaxiom/ghcr-cleanup-action@v1
|
||||
|
||||
@@ -8,12 +8,12 @@ on:
|
||||
jobs:
|
||||
build-artifacts:
|
||||
name: Build Artifacts
|
||||
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
|
||||
if: github.repository == 'kestra-io/kestra' # prevent running on forks
|
||||
uses: ./.github/workflows/workflow-build-artifacts.yml
|
||||
|
||||
publish:
|
||||
name: Publish Docker
|
||||
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
|
||||
if: github.repository == 'kestra-io/kestra' # prevent running on forks
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-artifacts
|
||||
env:
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
# Add comment on pull request
|
||||
- name: Add comment to PR
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
|
||||
14
.github/workflows/workflow-test.yml
vendored
14
.github/workflows/workflow-test.yml
vendored
@@ -84,14 +84,12 @@ jobs:
|
||||
name: Notify - Slack
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ frontend, backend ]
|
||||
if: github.event_name == 'schedule'
|
||||
steps:
|
||||
- name: Notify failed CI
|
||||
id: send-ci-failed
|
||||
if: |
|
||||
always() && (needs.frontend.result != 'success' ||
|
||||
needs.backend.result != 'success')
|
||||
uses: kestra-io/actions/.github/actions/send-ci-failed@main
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
always() &&
|
||||
(needs.frontend.result != 'success' || needs.backend.result != 'success') &&
|
||||
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
@@ -33,10 +33,10 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.kestra.io/video/product-overview" target="_blank">
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
|
||||
</a>
|
||||
</p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
|
||||
|
||||
|
||||
## 🌟 What is Kestra?
|
||||
|
||||
25
build.gradle
25
build.gradle
@@ -32,12 +32,12 @@ plugins {
|
||||
|
||||
// release
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.2"
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.3"
|
||||
id 'signing'
|
||||
id "com.vanniktech.maven.publish" version "0.34.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.3" apply false
|
||||
id "org.owasp.dependencycheck" version "12.1.5" apply false
|
||||
}
|
||||
|
||||
idea {
|
||||
@@ -168,8 +168,9 @@ allprojects {
|
||||
/**********************************************************************************************************************\
|
||||
* Test
|
||||
**********************************************************************************************************************/
|
||||
subprojects {
|
||||
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
|
||||
subprojects {subProj ->
|
||||
|
||||
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
|
||||
apply plugin: "com.adarshr.test-logger"
|
||||
|
||||
java {
|
||||
@@ -207,6 +208,13 @@ subprojects {
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
reports {
|
||||
junitXml.required = true
|
||||
junitXml.outputPerTestCase = true
|
||||
junitXml.mergeReruns = true
|
||||
junitXml.includeSystemErrLog = true;
|
||||
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
|
||||
}
|
||||
|
||||
// set Xmx for test workers
|
||||
maxHeapSize = '4g'
|
||||
@@ -222,6 +230,15 @@ subprojects {
|
||||
environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
|
||||
environment 'ENV_TEST1', "true"
|
||||
environment 'ENV_TEST2', "Pass by env"
|
||||
|
||||
|
||||
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
|
||||
// JUnit 5 parallel settings
|
||||
systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
|
||||
systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
|
||||
systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
|
||||
systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
|
||||
}
|
||||
}
|
||||
|
||||
testlogger {
|
||||
|
||||
@@ -40,5 +40,6 @@ dependencies {
|
||||
implementation project(":worker")
|
||||
|
||||
//test
|
||||
testImplementation project(':tests')
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
@@ -49,7 +49,7 @@ import java.util.concurrent.Callable;
|
||||
@Introspected
|
||||
public class App implements Callable<Integer> {
|
||||
public static void main(String[] args) {
|
||||
execute(App.class, args);
|
||||
execute(App.class, new String [] { Environment.CLI }, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
|
||||
return PicocliRunner.call(App.class, "--help");
|
||||
}
|
||||
|
||||
protected static void execute(Class<?> cls, String... args) {
|
||||
protected static void execute(Class<?> cls, String[] environments, String... args) {
|
||||
// Log Bridge
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
|
||||
// Init ApplicationContext
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, args);
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
|
||||
|
||||
// Call Picocli command
|
||||
int exitCode = 0;
|
||||
@@ -80,6 +80,7 @@ public class App implements Callable<Integer> {
|
||||
System.exit(Objects.requireNonNullElse(exitCode, 0));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
|
||||
* forced Properties from current command.
|
||||
@@ -88,12 +89,13 @@ public class App implements Callable<Integer> {
|
||||
* @return the application context created
|
||||
*/
|
||||
protected static ApplicationContext applicationContext(Class<?> mainClass,
|
||||
String[] environments,
|
||||
String[] args) {
|
||||
|
||||
ApplicationContextBuilder builder = ApplicationContext
|
||||
.builder()
|
||||
.mainClass(mainClass)
|
||||
.environments(Environment.CLI);
|
||||
.environments(environments);
|
||||
|
||||
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
|
||||
continueOnParsingErrors(cmd);
|
||||
|
||||
@@ -2,19 +2,27 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@Slf4j
|
||||
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
|
||||
Integer serverPort;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
|
||||
|
||||
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
|
||||
|
||||
return super.call();
|
||||
}
|
||||
|
||||
private long maxMemoryInMB() {
|
||||
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
|
||||
}
|
||||
|
||||
protected static int defaultWorkerThread() {
|
||||
return Runtime.getRuntime().availableProcessors() * 8;
|
||||
}
|
||||
|
||||
@@ -262,6 +262,8 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private String getTenantIdFromPath(Path path) {
|
||||
// FIXME there is probably a bug here when a tenant has '_' in its name,
|
||||
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
|
||||
return path.getFileName().toString().split("_")[0];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,6 +167,8 @@ kestra:
|
||||
open-urls:
|
||||
- "/ping"
|
||||
- "/api/v1/executions/webhook/"
|
||||
- "/api/v1/main/executions/webhook/"
|
||||
- "/api/v1/*/executions/webhook/"
|
||||
|
||||
preview:
|
||||
initial-rows: 100
|
||||
|
||||
@@ -37,7 +37,7 @@ class AppTest {
|
||||
|
||||
final String[] args = new String[]{"server", serverType, "--help"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
|
||||
|
||||
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
|
||||
@@ -52,7 +52,7 @@ class AppTest {
|
||||
|
||||
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
|
||||
|
||||
assertThat(out.toString()).startsWith("Missing required parameters: ");
|
||||
|
||||
@@ -4,11 +4,11 @@ import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -18,8 +18,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -57,10 +57,11 @@ class FileChangedEventListenerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
@Test
|
||||
void test() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a basic flow
|
||||
String flow = """
|
||||
@@ -73,14 +74,14 @@ class FileChangedEventListenerTest {
|
||||
message: Hello World! 🚀
|
||||
""";
|
||||
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
assertThat(myflow.getTasks()).hasSize(1);
|
||||
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
|
||||
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -88,16 +89,17 @@ class FileChangedEventListenerTest {
|
||||
// delete the flow
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
@RetryingTest(2)
|
||||
void testWithPluginDefault() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a flow with plugin default
|
||||
String pluginDefault = """
|
||||
@@ -113,14 +115,14 @@ class FileChangedEventListenerTest {
|
||||
values:
|
||||
message: Hello World!
|
||||
""";
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -128,7 +130,7 @@ class FileChangedEventListenerTest {
|
||||
// delete both files
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
|
||||
@@ -84,7 +84,7 @@ dependencies {
|
||||
|
||||
testImplementation "org.testcontainers:testcontainers:1.21.3"
|
||||
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on"
|
||||
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
|
||||
@@ -3,30 +3,88 @@ package io.kestra.core.events;
|
||||
import io.micronaut.core.annotation.Nullable;
|
||||
import io.micronaut.http.HttpRequest;
|
||||
import io.micronaut.http.context.ServerRequestContext;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
@AllArgsConstructor
|
||||
import java.util.Objects;
|
||||
|
||||
@Getter
|
||||
public class CrudEvent<T> {
|
||||
T model;
|
||||
private final T model;
|
||||
@Nullable
|
||||
T previousModel;
|
||||
CrudEventType type;
|
||||
HttpRequest<?> request;
|
||||
|
||||
private final T previousModel;
|
||||
private final CrudEventType type;
|
||||
private final HttpRequest<?> request;
|
||||
|
||||
/**
|
||||
* Static helper method for creating a new {@link CrudEventType#UPDATE} CrudEvent.
|
||||
*
|
||||
* @param model the new created model.
|
||||
* @param <T> type of the model.
|
||||
* @return the new {@link CrudEvent}.
|
||||
*/
|
||||
public static <T> CrudEvent<T> create(T model) {
|
||||
Objects.requireNonNull(model, "Can't create CREATE event with a null model");
|
||||
return new CrudEvent<>(model, null, CrudEventType.CREATE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper method for creating a new {@link CrudEventType#DELETE} CrudEvent.
|
||||
*
|
||||
* @param model the deleted model.
|
||||
* @param <T> type of the model.
|
||||
* @return the new {@link CrudEvent}.
|
||||
*/
|
||||
public static <T> CrudEvent<T> delete(T model) {
|
||||
Objects.requireNonNull(model, "Can't create DELETE event with a null model");
|
||||
return new CrudEvent<>(null, model, CrudEventType.DELETE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper method for creating a new CrudEvent.
|
||||
*
|
||||
* @param before the model before the update.
|
||||
* @param after the model after the update.
|
||||
* @param <T> type of the model.
|
||||
* @return the new {@link CrudEvent}.
|
||||
*/
|
||||
public static <T> CrudEvent<T> of(T before, T after) {
|
||||
|
||||
if (before == null && after == null) {
|
||||
throw new IllegalArgumentException("Both before and after cannot be null");
|
||||
}
|
||||
|
||||
if (before == null) {
|
||||
return create(after);
|
||||
}
|
||||
|
||||
if (after == null) {
|
||||
return delete(before);
|
||||
}
|
||||
|
||||
return new CrudEvent<>(after, before, CrudEventType.UPDATE);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated use the static factory methods.
|
||||
*/
|
||||
@Deprecated
|
||||
public CrudEvent(T model, CrudEventType type) {
|
||||
this.model = model;
|
||||
this.type = type;
|
||||
this.previousModel = null;
|
||||
this.request = ServerRequestContext.currentRequest().orElse(null);
|
||||
this(
|
||||
CrudEventType.DELETE.equals(type) ? null : model,
|
||||
CrudEventType.DELETE.equals(type) ? model : null,
|
||||
type,
|
||||
ServerRequestContext.currentRequest().orElse(null)
|
||||
);
|
||||
}
|
||||
|
||||
public CrudEvent(T model, T previousModel, CrudEventType type) {
|
||||
this(model, previousModel, type, ServerRequestContext.currentRequest().orElse(null));
|
||||
}
|
||||
|
||||
public CrudEvent(T model, T previousModel, CrudEventType type, HttpRequest<?> request) {
|
||||
this.model = model;
|
||||
this.previousModel = previousModel;
|
||||
this.type = type;
|
||||
this.request = ServerRequestContext.currentRequest().orElse(null);
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,12 +2,13 @@ package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public record Label(@NotNull String key, @NotNull String value) {
|
||||
public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static final String SYSTEM_PREFIX = "system.";
|
||||
|
||||
// system labels
|
||||
@@ -41,7 +42,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
@@ -56,6 +57,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
@@ -70,6 +72,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
if (map == null || map.isEmpty()) return List.of();
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
@@ -88,4 +91,14 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides predicate for not empty entries.
|
||||
*
|
||||
* @return The non-empty filter
|
||||
*/
|
||||
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
|
||||
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
|
||||
entry.getValue() != null && !entry.getValue().isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,33 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface that can be implemented by classes supporting plugin versioning.
|
||||
*
|
||||
* @see Plugin
|
||||
*/
|
||||
public interface PluginVersioning {
|
||||
|
||||
String TITLE = "Plugin Version";
|
||||
String DESCRIPTION = """
|
||||
Defines the version of the plugin to use.
|
||||
|
||||
@Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)")
|
||||
@Schema(title = "The version of the plugin to use.")
|
||||
The version must follow the Semantic Versioning (SemVer) specification:
|
||||
- A single-digit MAJOR version (e.g., `1`).
|
||||
- A MAJOR.MINOR version (e.g., `1.1`).
|
||||
- A MAJOR.MINOR.PATCH version, optionally with any qualifier
|
||||
(e.g., `1.1.2`, `1.1.0-SNAPSHOT`).
|
||||
""";
|
||||
|
||||
@Schema(
|
||||
title = TITLE,
|
||||
description = DESCRIPTION
|
||||
)
|
||||
String getVersion();
|
||||
}
|
||||
|
||||
@@ -254,19 +254,7 @@ public record QueryFilter(
|
||||
*
|
||||
* @return List of {@code ResourceField} with resource names, fields, and operations.
|
||||
*/
|
||||
public static List<ResourceField> asResourceList() {
|
||||
return Arrays.stream(values())
|
||||
.map(Resource::toResourceField)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ResourceField toResourceField(Resource resource) {
|
||||
List<FieldOp> fieldOps = resource.supportedField().stream()
|
||||
.map(Resource::toFieldInfo)
|
||||
.toList();
|
||||
return new ResourceField(resource.name().toLowerCase(), fieldOps);
|
||||
}
|
||||
|
||||
|
||||
private static FieldOp toFieldInfo(Field field) {
|
||||
List<Operation> operations = field.supportedOp().stream()
|
||||
.map(Resource::toOperation)
|
||||
@@ -279,9 +267,6 @@ public record QueryFilter(
|
||||
}
|
||||
}
|
||||
|
||||
public record ResourceField(String name, List<FieldOp> fields) {
|
||||
}
|
||||
|
||||
public record FieldOp(String name, String value, List<Operation> operations) {
|
||||
}
|
||||
|
||||
|
||||
@@ -17,31 +17,12 @@ import java.util.List;
|
||||
@Introspected
|
||||
public class ExecutionUsage {
|
||||
private final List<DailyExecutionStatistics> dailyExecutionsCount;
|
||||
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
|
||||
|
||||
public static ExecutionUsage of(final String tenantId,
|
||||
final ExecutionRepositoryInterface executionRepository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
|
||||
try {
|
||||
dailyTaskRunsCount = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenantId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
true);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
|
||||
}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(executionRepository.dailyStatistics(
|
||||
null,
|
||||
@@ -52,28 +33,13 @@ public class ExecutionUsage {
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
false))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
null))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
try {
|
||||
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
true
|
||||
);
|
||||
} catch (UnsupportedOperationException ignored) {}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
@@ -81,10 +47,8 @@ public class ExecutionUsage {
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
false
|
||||
DateUtils.GroupType.DAY
|
||||
))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -865,20 +865,18 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
* @param e the exception raise
|
||||
* @return new taskRun with updated attempt with logs
|
||||
*/
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
|
||||
TaskRunAttempt lastAttempt, Exception e) {
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
|
||||
TaskRun failed = taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
return new FailedTaskRunWithLog(
|
||||
taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt
|
||||
.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
)
|
||||
.withState(State.Type.FAILED),
|
||||
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
|
||||
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ public abstract class AbstractFlow implements FlowInterface {
|
||||
@JsonSerialize(using = ListOrMapOfLabelSerializer.class)
|
||||
@JsonDeserialize(using = ListOrMapOfLabelDeserializer.class)
|
||||
@Schema(implementation = Object.class, oneOf = {List.class, Map.class})
|
||||
@Valid
|
||||
List<Label> labels;
|
||||
|
||||
@Schema(additionalProperties = Schema.AdditionalPropertiesValue.TRUE)
|
||||
|
||||
@@ -185,34 +185,6 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Trigger update(Trigger currentTrigger, Trigger newTrigger, ZonedDateTime nextExecutionDate) throws Exception {
|
||||
Trigger updated = currentTrigger;
|
||||
|
||||
// If a backfill is created, we update the currentTrigger
|
||||
// and set the nextExecutionDate() as the previous one
|
||||
if (newTrigger.getBackfill() != null) {
|
||||
updated = currentTrigger.toBuilder()
|
||||
.backfill(
|
||||
newTrigger
|
||||
.getBackfill()
|
||||
.toBuilder()
|
||||
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
|
||||
.currentDate(
|
||||
newTrigger.getBackfill().getStart()
|
||||
)
|
||||
.previousNextExecutionDate(
|
||||
currentTrigger.getNextExecutionDate())
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
return updated.toBuilder()
|
||||
.nextExecutionDate(newTrigger.getDisabled() ?
|
||||
null : nextExecutionDate)
|
||||
.disabled(newTrigger.getDisabled())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Trigger resetExecution(Flow flow, Execution execution, ConditionContext conditionContext) {
|
||||
boolean disabled = this.getStopAfter() != null ? this.getStopAfter().contains(execution.getState().getCurrent()) : this.getDisabled();
|
||||
if (!disabled) {
|
||||
@@ -276,27 +248,22 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
.build();
|
||||
}
|
||||
|
||||
public Trigger initBackfill(Trigger newTrigger) {
|
||||
// If a backfill is created, we update the currentTrigger
|
||||
public Trigger withBackfill(final Backfill backfill) {
|
||||
Trigger updated = this;
|
||||
// If a backfill is created, we update the trigger
|
||||
// and set the nextExecutionDate() as the previous one
|
||||
if (newTrigger.getBackfill() != null) {
|
||||
|
||||
return this.toBuilder()
|
||||
if (backfill != null) {
|
||||
updated = this.toBuilder()
|
||||
.backfill(
|
||||
newTrigger
|
||||
.getBackfill()
|
||||
backfill
|
||||
.toBuilder()
|
||||
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
|
||||
.currentDate(
|
||||
newTrigger.getBackfill().getStart()
|
||||
)
|
||||
.previousNextExecutionDate(
|
||||
this.getNextExecutionDate())
|
||||
.end(backfill.getEnd() != null ? backfill.getEnd() : ZonedDateTime.now())
|
||||
.currentDate(backfill.getStart())
|
||||
.previousNextExecutionDate(this.getNextExecutionDate())
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
return this;
|
||||
return updated;
|
||||
}
|
||||
|
||||
// if the next date is after the backfill end, we remove the backfill
|
||||
|
||||
@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
*
|
||||
* @return the {@link DefaultPluginRegistry}.
|
||||
*/
|
||||
public static DefaultPluginRegistry getOrCreate() {
|
||||
public synchronized static DefaultPluginRegistry getOrCreate() {
|
||||
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
|
||||
if (!instance.isInitialized()) {
|
||||
instance.init();
|
||||
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
/**
|
||||
* Initializes the registry by loading all core plugins.
|
||||
*/
|
||||
protected void init() {
|
||||
protected synchronized void init() {
|
||||
if (initialized.compareAndSet(false, true)) {
|
||||
register(scanner.scan());
|
||||
}
|
||||
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
if (existing != null && existing.crc32() == plugin.crc32()) {
|
||||
return; // same plugin already registered
|
||||
}
|
||||
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
if (existing != null) {
|
||||
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
|
||||
pluginClassByIdentifier.putAll(plugins);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,12 @@ import lombok.Getter;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Enumeration;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarFile;
|
||||
import java.util.zip.CRC32;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Getter
|
||||
@@ -14,5 +20,59 @@ import java.net.URL;
|
||||
public class ExternalPlugin {
|
||||
private final URL location;
|
||||
private final URL[] resources;
|
||||
private final long crc32;
|
||||
private volatile Long crc32; // lazy-val
|
||||
|
||||
public ExternalPlugin(URL location, URL[] resources) {
|
||||
this.location = location;
|
||||
this.resources = resources;
|
||||
}
|
||||
|
||||
public Long getCrc32() {
|
||||
if (this.crc32 == null) {
|
||||
synchronized (this) {
|
||||
if (this.crc32 == null) {
|
||||
this.crc32 = computeJarCrc32(location);
|
||||
}
|
||||
}
|
||||
}
|
||||
return crc32;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a CRC32 of the JAR File without reading the whole file
|
||||
*
|
||||
* @param location of the JAR File.
|
||||
* @return the CRC32 of {@code -1} if the checksum can't be computed.
|
||||
*/
|
||||
private static long computeJarCrc32(final URL location) {
|
||||
CRC32 crc = new CRC32();
|
||||
try (JarFile jar = new JarFile(location.toURI().getPath(), false)) {
|
||||
Enumeration<JarEntry> entries = jar.entries();
|
||||
byte[] buffer = new byte[Long.BYTES]; // reusable buffer to avoid re-allocation
|
||||
|
||||
while (entries.hasMoreElements()) {
|
||||
JarEntry entry = entries.nextElement();
|
||||
crc.update(entry.getName().getBytes(StandardCharsets.UTF_8));
|
||||
updateCrc32WithLong(crc, buffer, entry.getSize());
|
||||
updateCrc32WithLong(crc, buffer, entry.getCrc());
|
||||
}
|
||||
|
||||
return crc.getValue();
|
||||
} catch (Exception e) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
private static void updateCrc32WithLong(CRC32 crc32, byte[] reusable, long val) {
|
||||
// fast long -> byte conversion
|
||||
reusable[0] = (byte) (val >>> 56);
|
||||
reusable[1] = (byte) (val >>> 48);
|
||||
reusable[2] = (byte) (val >>> 40);
|
||||
reusable[3] = (byte) (val >>> 32);
|
||||
reusable[4] = (byte) (val >>> 24);
|
||||
reusable[5] = (byte) (val >>> 16);
|
||||
reusable[6] = (byte) (val >>> 8);
|
||||
reusable[7] = (byte) val;
|
||||
crc32.update(reusable);;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ public class PluginClassLoader extends URLClassLoader {
|
||||
+ "|dev.failsafe"
|
||||
+ "|reactor"
|
||||
+ "|io.opentelemetry"
|
||||
+ "|io.netty"
|
||||
+ ")\\..*$");
|
||||
|
||||
private final ClassLoader parent;
|
||||
|
||||
@@ -51,8 +51,7 @@ public class PluginResolver {
|
||||
final List<URL> resources = resolveUrlsForPluginPath(path);
|
||||
plugins.add(new ExternalPlugin(
|
||||
path.toUri().toURL(),
|
||||
resources.toArray(new URL[0]),
|
||||
computeJarCrc32(path)
|
||||
resources.toArray(new URL[0])
|
||||
));
|
||||
}
|
||||
} catch (final InvalidPathException | MalformedURLException e) {
|
||||
@@ -124,33 +123,5 @@ public class PluginResolver {
|
||||
|
||||
return urls;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Compute a CRC32 of the JAR File without reading the whole file
|
||||
*
|
||||
* @param location of the JAR File.
|
||||
* @return the CRC32 of {@code -1} if the checksum can't be computed.
|
||||
*/
|
||||
private static long computeJarCrc32(final Path location) {
|
||||
CRC32 crc = new CRC32();
|
||||
try (JarFile jar = new JarFile(location.toFile(), false)) {
|
||||
Enumeration<JarEntry> entries = jar.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
JarEntry entry = entries.nextElement();
|
||||
crc.update(entry.getName().getBytes());
|
||||
crc.update(longToBytes(entry.getSize()));
|
||||
crc.update(longToBytes(entry.getCrc()));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return -1;
|
||||
}
|
||||
return crc.getValue();
|
||||
}
|
||||
|
||||
private static byte[] longToBytes(long x) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
|
||||
buffer.putLong(x);
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
|
||||
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
|
||||
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::asText).orElse(null);
|
||||
|
||||
if (type == null || type.isEmpty()) {
|
||||
return null;
|
||||
|
||||
@@ -25,8 +25,6 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
|
||||
Boolean isTaskRunEnabled();
|
||||
|
||||
default Optional<Execution> findById(String tenantId, String id) {
|
||||
return findById(tenantId, id, false);
|
||||
}
|
||||
@@ -96,12 +94,6 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
|
||||
Flux<Execution> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
ArrayListTotal<TaskRun> findTaskRun(
|
||||
Pageable pageable,
|
||||
@Nullable String tenantId,
|
||||
List<QueryFilter> filters
|
||||
);
|
||||
|
||||
Execution delete(Execution execution);
|
||||
|
||||
Integer purge(Execution execution);
|
||||
@@ -112,8 +104,7 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
boolean isTaskRun
|
||||
@Nullable DateUtils.GroupType groupBy
|
||||
);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatistics(
|
||||
@@ -125,8 +116,7 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
List<State.Type> state,
|
||||
boolean isTaskRun
|
||||
List<State.Type> state
|
||||
);
|
||||
|
||||
@Getter
|
||||
|
||||
@@ -83,7 +83,9 @@ public class LocalFlowRepositoryLoader {
|
||||
}
|
||||
|
||||
public void load(String tenantId, File basePath) throws IOException {
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
|
||||
.stream()
|
||||
.filter(flow -> tenantId.equals(flow.getTenantId()))
|
||||
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
|
||||
|
||||
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {
|
||||
|
||||
@@ -5,10 +5,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.exceptions.InternalException;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
@@ -29,6 +26,7 @@ import org.apache.commons.lang3.stream.Streams;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.trace.Tracer.throwCallable;
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
@@ -153,17 +151,24 @@ public final class ExecutableUtils {
|
||||
currentFlow.getNamespace(),
|
||||
currentFlow.getId()
|
||||
)
|
||||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
|
||||
.orElseThrow(() -> {
|
||||
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
|
||||
runContext.logger().error(msg);
|
||||
return new IllegalStateException(msg);
|
||||
});
|
||||
|
||||
if (flow.isDisabled()) {
|
||||
throw new IllegalStateException("Cannot execute a flow which is disabled");
|
||||
String msg = "Cannot execute a flow which is disabled";
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
}
|
||||
|
||||
if (flow instanceof FlowWithException fwe) {
|
||||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
|
||||
String msg = "Cannot execute an invalid flow: " + fwe.getException();
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
}
|
||||
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
if (labels != null) {
|
||||
labels.forEach(throwConsumer(label -> newLabels.add(new Label(runContext.render(label.key()), runContext.render(label.value())))));
|
||||
}
|
||||
@@ -201,7 +206,20 @@ public final class ExecutableUtils {
|
||||
.build()
|
||||
)
|
||||
.withScheduleDate(scheduleOnDate);
|
||||
|
||||
if(execution.getInputs().size()<inputs.size()) {
|
||||
Map<String,Object>resolvedInputs=execution.getInputs();
|
||||
for (var inputKey : inputs.keySet()) {
|
||||
if (!resolvedInputs.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
|
||||
inputKey,
|
||||
currentExecution.getId(),
|
||||
currentTask.subflowId().namespace(),
|
||||
currentTask.subflowId().flowId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// inject the traceparent into the new execution
|
||||
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));
|
||||
|
||||
|
||||
@@ -49,15 +49,7 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -231,6 +223,19 @@ public class FlowInputOutput {
|
||||
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
|
||||
})
|
||||
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||
if (resolved.size() < data.size()) {
|
||||
RunContext runContext = runContextFactory.of(flow, execution);
|
||||
for (var inputKey : data.keySet()) {
|
||||
if (!resolved.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
|
||||
inputKey,
|
||||
flow.getNamespace(),
|
||||
flow.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return MapUtils.flattenToNestedMap(resolved);
|
||||
}
|
||||
|
||||
@@ -313,15 +318,15 @@ public class FlowInputOutput {
|
||||
});
|
||||
resolvable.setInput(input);
|
||||
|
||||
|
||||
|
||||
Object value = resolvable.get().value();
|
||||
|
||||
|
||||
// resolve default if needed
|
||||
if (value == null && input.getDefaults() != null) {
|
||||
value = resolveDefaultValue(input, runContext);
|
||||
resolvable.isDefault(true);
|
||||
}
|
||||
|
||||
|
||||
// validate and parse input value
|
||||
if (value == null) {
|
||||
if (input.getRequired()) {
|
||||
@@ -350,7 +355,7 @@ public class FlowInputOutput {
|
||||
|
||||
return resolvable.get();
|
||||
}
|
||||
|
||||
|
||||
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
|
||||
return switch (input.getType()) {
|
||||
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
|
||||
@@ -367,7 +372,7 @@ public class FlowInputOutput {
|
||||
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
|
||||
@@ -376,7 +381,7 @@ public class FlowInputOutput {
|
||||
private static <T> Object resolveDefaultPropertyAsList(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.asList((Property<List<T>>) input.getDefaults(), renderer, clazz);
|
||||
}
|
||||
|
||||
|
||||
private RunContext buildRunContextForExecutionAndInputs(final FlowInterface flow, final Execution execution, Map<String, InputAndValue> dependencies) {
|
||||
Map<String, Object> flattenInputs = MapUtils.flattenToNestedMap(dependencies.entrySet()
|
||||
.stream()
|
||||
@@ -453,7 +458,7 @@ public class FlowInputOutput {
|
||||
if (data.getType() == null) {
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
|
||||
}
|
||||
|
||||
|
||||
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
|
||||
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(
|
||||
@@ -530,17 +535,17 @@ public class FlowInputOutput {
|
||||
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
if (outputs == null) return Map.of();
|
||||
|
||||
|
||||
// render required outputs
|
||||
Map<String, Object> outputsById = outputs
|
||||
.stream()
|
||||
.filter(output -> output.getRequired() == null || output.getRequired())
|
||||
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
|
||||
outputsById = runContext.render(outputsById);
|
||||
|
||||
|
||||
// render optional outputs one by one to catch, log, and skip any error.
|
||||
for (io.kestra.core.models.flows.Output output : outputs) {
|
||||
if (Boolean.FALSE.equals(output.getRequired())) {
|
||||
@@ -583,9 +588,9 @@ public class FlowInputOutput {
|
||||
}
|
||||
|
||||
public void isDefault(boolean isDefault) {
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
}
|
||||
|
||||
|
||||
public void setInput(final Input<?> input) {
|
||||
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
|
||||
}
|
||||
|
||||
@@ -500,7 +500,7 @@ public class FlowableUtils {
|
||||
|
||||
ArrayList<ResolvedTask> result = new ArrayList<>();
|
||||
|
||||
int index = 0;
|
||||
int iteration = 0;
|
||||
for (Object current : distinctValue) {
|
||||
try {
|
||||
String resolvedValue = current instanceof String stringValue ? stringValue : MAPPER.writeValueAsString(current);
|
||||
@@ -508,7 +508,7 @@ public class FlowableUtils {
|
||||
result.add(ResolvedTask.builder()
|
||||
.task(task)
|
||||
.value(resolvedValue)
|
||||
.iteration(index++)
|
||||
.iteration(iteration)
|
||||
.parentId(parentTaskRun.getId())
|
||||
.build()
|
||||
);
|
||||
@@ -516,6 +516,7 @@ public class FlowableUtils {
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalVariableEvaluationException(e);
|
||||
}
|
||||
iteration++;
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -10,6 +10,7 @@ import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.Input;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.input.SecretInput;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.property.PropertyContext;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
@@ -282,15 +283,15 @@ public final class RunVariables {
|
||||
|
||||
if (flow != null && flow.getInputs() != null) {
|
||||
// we add default inputs value from the flow if not already set, this will be useful for triggers
|
||||
flow.getInputs().stream()
|
||||
.filter(input -> input.getDefaults() != null && !inputs.containsKey(input.getId()))
|
||||
.forEach(input -> {
|
||||
try {
|
||||
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, propertyContext));
|
||||
} catch (IllegalVariableEvaluationException e) {
|
||||
throw new RuntimeException("Unable to inject default value for input '" + input.getId() + "'", e);
|
||||
}
|
||||
});
|
||||
flow.getInputs().stream()
|
||||
.filter(input -> input.getDefaults() != null && !inputs.containsKey(input.getId()))
|
||||
.forEach(input -> {
|
||||
try {
|
||||
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, propertyContext));
|
||||
} catch (IllegalVariableEvaluationException e) {
|
||||
// Silent catch, if an input depends on another input, or a variable that is populated at runtime / input filling time, we can't resolve it here.
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (!inputs.isEmpty()) {
|
||||
|
||||
@@ -45,7 +45,7 @@ final class Secret {
|
||||
for (var entry: data.entrySet()) {
|
||||
if (entry.getValue() instanceof Map map) {
|
||||
// if some value are of type EncryptedString we decode them and replace the object
|
||||
if (EncryptedString.TYPE.equalsIgnoreCase((String)map.get("type"))) {
|
||||
if (map.get("type") instanceof String typeStr && EncryptedString.TYPE.equalsIgnoreCase(typeStr)) {
|
||||
try {
|
||||
String decoded = decrypt((String) map.get("value"));
|
||||
decryptedMap.put(entry.getKey(), decoded);
|
||||
|
||||
@@ -168,6 +168,7 @@ public class Extension extends AbstractExtension {
|
||||
functions.put("randomPort", new RandomPortFunction());
|
||||
functions.put("fileExists", fileExistsFunction);
|
||||
functions.put("isFileEmpty", isFileEmptyFunction);
|
||||
functions.put("nanoId", new NanoIDFunction());
|
||||
functions.put("tasksWithState", new TasksWithStateFunction());
|
||||
functions.put(HttpFunction.NAME, httpFunction);
|
||||
return functions;
|
||||
|
||||
@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
|
||||
ZoneId zoneId = zoneId(timeZone);
|
||||
ZonedDateTime date = convert(input, zoneId, existingFormat);
|
||||
|
||||
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import io.kestra.core.http.HttpRequest;
|
||||
import io.kestra.core.http.HttpResponse;
|
||||
import io.kestra.core.http.client.HttpClient;
|
||||
import io.kestra.core.http.client.HttpClientException;
|
||||
import io.kestra.core.http.client.HttpClientRequestException;
|
||||
import io.kestra.core.http.client.HttpClientResponseException;
|
||||
import io.kestra.core.http.client.configurations.HttpConfiguration;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
@@ -101,8 +103,15 @@ public class HttpFunction<T> implements Function {
|
||||
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
|
||||
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
|
||||
return response.getBody();
|
||||
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
|
||||
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
|
||||
} catch (HttpClientResponseException e) {
|
||||
if (e.getResponse() != null) {
|
||||
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
|
||||
throw new PebbleException(e, msg , lineNumber, self.getName());
|
||||
} else {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package io.kestra.core.runners.pebble.functions;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Function;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NanoIDFunction implements Function {
|
||||
|
||||
private static final int DEFAULT_LENGTH = 21;
|
||||
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
private static final String LENGTH = "length";
|
||||
private static final String ALPHABET = "alphabet";
|
||||
|
||||
private static final int MAX_LENGTH = 1000;
|
||||
|
||||
@Override
|
||||
public Object execute(
|
||||
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
|
||||
int length = DEFAULT_LENGTH;
|
||||
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
|
||||
length = parseLength(args, self, lineNumber);
|
||||
}
|
||||
char[] alphabet = DEFAULT_ALPHABET;
|
||||
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
|
||||
alphabet = ((String) args.get(ALPHABET)).toCharArray();
|
||||
}
|
||||
return createNanoID(length, alphabet);
|
||||
}
|
||||
|
||||
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
|
||||
var value = (Long) args.get(LENGTH);
|
||||
if(value > MAX_LENGTH) {
|
||||
throw new PebbleException(
|
||||
null,
|
||||
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
|
||||
lineNumber,
|
||||
self.getName());
|
||||
}
|
||||
return Math.toIntExact(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
return List.of(LENGTH,ALPHABET);
|
||||
}
|
||||
|
||||
String createNanoID(int length, char[] alphabet){
|
||||
final char[] data = new char[length];
|
||||
final byte[] bytes = new byte[length];
|
||||
final int mask = alphabet.length-1;
|
||||
secureRandom.nextBytes(bytes);
|
||||
for (int i = 0; i < length; ++i) {
|
||||
data[i] = alphabet[bytes[i] & mask];
|
||||
}
|
||||
return String.valueOf(data);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -180,23 +180,13 @@ public final class FileSerde {
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,31 +163,28 @@ public final class JacksonMapper {
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Pair<JsonNode, JsonNode> getBiDirectionalDiffs(Object previous, Object current) {
|
||||
JsonNode previousJson = MAPPER.valueToTree(previous);
|
||||
JsonNode newJson = MAPPER.valueToTree(current);
|
||||
public static Pair<JsonNode, JsonNode> getBiDirectionalDiffs(Object before, Object after) {
|
||||
JsonNode beforeNode = MAPPER.valueToTree(before);
|
||||
JsonNode afterNode = MAPPER.valueToTree(after);
|
||||
|
||||
JsonNode patchPrevToNew = JsonDiff.asJson(previousJson, newJson);
|
||||
JsonNode patchNewToPrev = JsonDiff.asJson(newJson, previousJson);
|
||||
JsonNode patch = JsonDiff.asJson(beforeNode, afterNode);
|
||||
JsonNode revert = JsonDiff.asJson(afterNode, beforeNode);
|
||||
|
||||
return Pair.of(patchPrevToNew, patchNewToPrev);
|
||||
return Pair.of(patch, revert);
|
||||
}
|
||||
|
||||
public static String applyPatches(Object object, List<JsonNode> patches) throws JsonProcessingException {
|
||||
|
||||
public static JsonNode applyPatchesOnJsonNode(JsonNode jsonObject, List<JsonNode> patches) {
|
||||
for (JsonNode patch : patches) {
|
||||
try {
|
||||
// Required for ES
|
||||
if (patch.findValue("value") == null) {
|
||||
((ObjectNode) patch.get(0)).set("value", (JsonNode) null);
|
||||
if (patch.findValue("value") == null && !patch.isEmpty()) {
|
||||
((ObjectNode) patch.get(0)).set("value", null);
|
||||
}
|
||||
JsonNode current = MAPPER.valueToTree(object);
|
||||
object = JsonPatch.fromJson(patch).apply(current);
|
||||
jsonObject = JsonPatch.fromJson(patch).apply(jsonObject);
|
||||
} catch (IOException | JsonPatchException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
return MAPPER.writeValueAsString(object);
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -3,12 +3,7 @@ package io.kestra.core.services;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import io.kestra.core.exceptions.FlowProcessingException;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowId;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.tasks.RunnableTask;
|
||||
import io.kestra.core.models.topologies.FlowTopology;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
@@ -30,16 +25,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
@@ -551,23 +537,24 @@ public class FlowService {
|
||||
return expandAll ? recursiveFlowTopology(new ArrayList<>(), tenant, namespace, id, destinationOnly) : flowTopologyRepository.get().findByFlow(tenant, namespace, id, destinationOnly).stream();
|
||||
}
|
||||
|
||||
private Stream<FlowTopology> recursiveFlowTopology(List<FlowId> flowIds, String tenantId, String namespace, String id, boolean destinationOnly) {
|
||||
private Stream<FlowTopology> recursiveFlowTopology(List<String> visitedTopologies, String tenantId, String namespace, String id, boolean destinationOnly) {
|
||||
if (flowTopologyRepository.isEmpty()) {
|
||||
throw noRepositoryException();
|
||||
}
|
||||
|
||||
List<FlowTopology> flowTopologies = flowTopologyRepository.get().findByFlow(tenantId, namespace, id, destinationOnly);
|
||||
|
||||
FlowId flowId = FlowId.of(tenantId, namespace, id, null);
|
||||
if (flowIds.contains(flowId)) {
|
||||
return flowTopologies.stream();
|
||||
}
|
||||
flowIds.add(flowId);
|
||||
var flowTopologies = flowTopologyRepository.get().findByFlow(tenantId, namespace, id, destinationOnly);
|
||||
|
||||
return flowTopologies.stream()
|
||||
.flatMap(topology -> Stream.of(topology.getDestination(), topology.getSource()))
|
||||
// recursively fetch child nodes
|
||||
.flatMap(node -> recursiveFlowTopology(flowIds, node.getTenantId(), node.getNamespace(), node.getId(), destinationOnly));
|
||||
// ignore already visited topologies
|
||||
.filter(x -> !visitedTopologies.contains(x.uid()))
|
||||
.flatMap(topology -> {
|
||||
visitedTopologies.add(topology.uid());
|
||||
Stream<FlowTopology> subTopologies = Stream
|
||||
.of(topology.getDestination(), topology.getSource())
|
||||
// recursively visit children and parents nodes
|
||||
.flatMap(relationNode -> recursiveFlowTopology(visitedTopologies, relationNode.getTenantId(), relationNode.getNamespace(), relationNode.getId(), destinationOnly));
|
||||
return Stream.concat(Stream.of(topology), subTopologies);
|
||||
});
|
||||
}
|
||||
|
||||
private IllegalStateException noRepositoryException() {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package io.kestra.core.storages.kv;
|
||||
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.time.Duration;
|
||||
@@ -9,6 +10,7 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
@Getter
|
||||
@EqualsAndHashCode
|
||||
public class KVMetadata {
|
||||
private String description;
|
||||
private Instant expirationDate;
|
||||
@@ -17,14 +19,18 @@ public class KVMetadata {
|
||||
if (ttl != null && ttl.isNegative()) {
|
||||
throw new IllegalArgumentException("ttl cannot be negative");
|
||||
}
|
||||
|
||||
|
||||
|
||||
this.description = description;
|
||||
if (ttl != null) {
|
||||
this.expirationDate = Instant.now().plus(ttl);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public KVMetadata(String description, Instant expirationDate) {
|
||||
this.description = description;
|
||||
this.expirationDate = expirationDate;
|
||||
}
|
||||
|
||||
public KVMetadata(Map<String, String> metadata) {
|
||||
if (metadata == null) {
|
||||
return;
|
||||
@@ -46,4 +52,9 @@ public class KVMetadata {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "[description=" + description + ", expirationDate=" + expirationDate + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@ import io.kestra.core.exceptions.ResourceExpiredException;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -104,8 +106,33 @@ public interface KVStore {
|
||||
default boolean exists(String key) throws IOException {
|
||||
return list().stream().anyMatch(kvEntry -> kvEntry.key().equals(key));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Finds a KV entry with associated metadata for a given key.
|
||||
*
|
||||
* @param key the KV entry key.
|
||||
* @return an optional of {@link KVValueAndMetadata}.
|
||||
*
|
||||
* @throws UncheckedIOException if an error occurred while executing the operation on the K/V store.
|
||||
*/
|
||||
default Optional<KVValueAndMetadata> findMetadataAndValue(final String key) throws UncheckedIOException {
|
||||
try {
|
||||
return get(key).flatMap(entry ->
|
||||
{
|
||||
try {
|
||||
return getValue(entry.key()).map(current -> new KVValueAndMetadata(new KVMetadata(entry.description(), entry.expirationDate()), current.value()));
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
} catch (ResourceExpiredException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
Pattern KEY_VALIDATOR_PATTERN = Pattern.compile("[a-zA-Z0-9][a-zA-Z0-9._-]*");
|
||||
|
||||
/**
|
||||
|
||||
@@ -18,6 +18,7 @@ import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.FlowTopologyRepositoryInterface;
|
||||
import io.kestra.core.services.ConditionService;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import io.kestra.plugin.core.condition.*;
|
||||
import io.micronaut.core.annotation.Nullable;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -175,9 +176,6 @@ public class FlowTopologyService {
|
||||
protected boolean isTriggerChild(Flow parent, Flow child) {
|
||||
List<AbstractTrigger> triggers = ListUtils.emptyOnNull(child.getTriggers());
|
||||
|
||||
// simulated execution: we add a "simulated" label so conditions can know that the evaluation is for a simulated execution
|
||||
Execution execution = Execution.newExecution(parent, (f, e) -> null, List.of(SIMULATED_EXECUTION), Optional.empty());
|
||||
|
||||
// keep only flow trigger
|
||||
List<io.kestra.plugin.core.trigger.Flow> flowTriggers = triggers
|
||||
.stream()
|
||||
@@ -189,13 +187,16 @@ public class FlowTopologyService {
|
||||
return false;
|
||||
}
|
||||
|
||||
// simulated execution: we add a "simulated" label so conditions can know that the evaluation is for a simulated execution
|
||||
Execution execution = Execution.newExecution(parent, (f, e) -> null, List.of(SIMULATED_EXECUTION), Optional.empty());
|
||||
|
||||
boolean conditionMatch = flowTriggers
|
||||
.stream()
|
||||
.flatMap(flow -> ListUtils.emptyOnNull(flow.getConditions()).stream())
|
||||
.allMatch(condition -> validateCondition(condition, parent, execution));
|
||||
|
||||
boolean preconditionMatch = flowTriggers.stream()
|
||||
.anyMatch(flow -> flow.getPreconditions() == null || validateMultipleConditions(flow.getPreconditions().getConditions(), parent, execution));
|
||||
.anyMatch(flow -> flow.getPreconditions() == null || validatePreconditions(flow.getPreconditions(), parent, execution));
|
||||
|
||||
return conditionMatch && preconditionMatch;
|
||||
}
|
||||
@@ -239,11 +240,24 @@ public class FlowTopologyService {
|
||||
}
|
||||
|
||||
private boolean isMandatoryMultipleCondition(Condition condition) {
|
||||
return Stream
|
||||
.of(
|
||||
Expression.class
|
||||
)
|
||||
.anyMatch(aClass -> condition.getClass().isAssignableFrom(aClass));
|
||||
return condition.getClass().isAssignableFrom(Expression.class);
|
||||
}
|
||||
|
||||
private boolean validatePreconditions(io.kestra.plugin.core.trigger.Flow.Preconditions preconditions, FlowInterface child, Execution execution) {
|
||||
boolean upstreamFlowMatched = MapUtils.emptyOnNull(preconditions.getUpstreamFlowsConditions())
|
||||
.values()
|
||||
.stream()
|
||||
.filter(c -> !isFilterCondition(c))
|
||||
.anyMatch(c -> validateCondition(c, child, execution));
|
||||
|
||||
boolean whereMatched = MapUtils.emptyOnNull(preconditions.getWhereConditions())
|
||||
.values()
|
||||
.stream()
|
||||
.filter(c -> !isFilterCondition(c))
|
||||
.allMatch(c -> validateCondition(c, child, execution));
|
||||
|
||||
// to be a dependency, if upstream flow is set it must be either inside it so it's a AND between upstream flow and where
|
||||
return upstreamFlowMatched && whereMatched;
|
||||
}
|
||||
|
||||
private boolean isFilterCondition(Condition condition) {
|
||||
|
||||
@@ -206,22 +206,17 @@ public class MapUtils {
|
||||
|
||||
/**
|
||||
* Utility method that flatten a nested map.
|
||||
* <p>
|
||||
* NOTE: for simplicity, this method didn't allow to flatten maps with conflicting keys that would end up in different flatten keys,
|
||||
* this could be related later if needed by flattening {k1: k2: {k3: v1}, k1: {k4: v2}} to {k1.k2.k3: v1, k1.k4: v2} is prohibited for now.
|
||||
*
|
||||
* @param nestedMap the nested map.
|
||||
* @return the flattened map.
|
||||
*
|
||||
* @throws IllegalArgumentException if any entry contains a map of more than one element.
|
||||
*/
|
||||
public static Map<String, Object> nestedToFlattenMap(@NotNull Map<String, Object> nestedMap) {
|
||||
Map<String, Object> result = new TreeMap<>();
|
||||
|
||||
for (Map.Entry<String, Object> entry : nestedMap.entrySet()) {
|
||||
if (entry.getValue() instanceof Map<?, ?> map) {
|
||||
Map.Entry<String, Object> flatten = flattenEntry(entry.getKey(), (Map<String, Object>) map);
|
||||
result.put(flatten.getKey(), flatten.getValue());
|
||||
Map<String, Object> flatten = flattenEntry(entry.getKey(), (Map<String, Object>) map);
|
||||
result.putAll(flatten);
|
||||
} else {
|
||||
result.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
@@ -229,18 +224,19 @@ public class MapUtils {
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Map.Entry<String, Object> flattenEntry(String key, Map<String, Object> value) {
|
||||
if (value.size() > 1) {
|
||||
throw new IllegalArgumentException("You cannot flatten a map with an entry that is a map of more than one element, conflicting key: " + key);
|
||||
private static Map<String, Object> flattenEntry(String key, Map<String, Object> value) {
|
||||
Map<String, Object> result = new TreeMap<>();
|
||||
|
||||
for (Map.Entry<String, Object> entry : value.entrySet()) {
|
||||
String newKey = key + "." + entry.getKey();
|
||||
Object newValue = entry.getValue();
|
||||
if (newValue instanceof Map<?, ?> map) {
|
||||
result.putAll(flattenEntry(newKey, (Map<String, Object>) map));
|
||||
} else {
|
||||
result.put(newKey, newValue);
|
||||
}
|
||||
}
|
||||
|
||||
Map.Entry<String, Object> entry = value.entrySet().iterator().next();
|
||||
String newKey = key + "." + entry.getKey();
|
||||
Object newValue = entry.getValue();
|
||||
if (newValue instanceof Map<?, ?> map) {
|
||||
return flattenEntry(newKey, (Map<String, Object>) map);
|
||||
} else {
|
||||
return Map.entry(newKey, newValue);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,48 +32,84 @@ public class Version implements Comparable<Version> {
|
||||
* @param version the version.
|
||||
* @return a new {@link Version} instance.
|
||||
*/
|
||||
public static Version of(String version) {
|
||||
public static Version of(final Object version) {
|
||||
|
||||
if (version.startsWith("v")) {
|
||||
version = version.substring(1);
|
||||
if (Objects.isNull(version)) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse null version");
|
||||
}
|
||||
|
||||
String strVersion = version.toString();
|
||||
|
||||
if (strVersion.startsWith("v")) {
|
||||
strVersion = strVersion.substring(1);
|
||||
}
|
||||
|
||||
int qualifier = version.indexOf("-");
|
||||
int qualifier = strVersion.indexOf("-");
|
||||
|
||||
final String[] versions = qualifier > 0 ?
|
||||
version.substring(0, qualifier).split("\\.") :
|
||||
version.split("\\.");
|
||||
strVersion.substring(0, qualifier).split("\\.") :
|
||||
strVersion.split("\\.");
|
||||
try {
|
||||
final int majorVersion = Integer.parseInt(versions[0]);
|
||||
final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0;
|
||||
final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0;
|
||||
final Integer minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : null;
|
||||
final Integer incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : null;
|
||||
|
||||
return new Version(
|
||||
majorVersion,
|
||||
minorVersion,
|
||||
incrementalVersion,
|
||||
qualifier > 0 ? version.substring(qualifier + 1) : null,
|
||||
version
|
||||
qualifier > 0 ? strVersion.substring(qualifier + 1) : null,
|
||||
strVersion
|
||||
);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Static helper method for returning the most recent stable version for a current {@link Version}.
|
||||
* Resolves the most appropriate stable version from a collection, based on a given input version.
|
||||
* <p>
|
||||
* The matching rules are:
|
||||
* <ul>
|
||||
* <li>If {@code from} specifies only a major version (e.g. {@code 1}), return the latest stable version
|
||||
* with the same major (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a major and minor version only (e.g. {@code 1.2}), return the latest
|
||||
* stable version with the same major and minor (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a full version with major, minor, and patch (e.g. {@code 1.2.2}),
|
||||
* then only return it if it is exactly present (and stable) in {@code versions}.
|
||||
* No "upgrade" is performed in this case.</li>
|
||||
* <li>If no suitable version is found, returns {@code null}.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param from the current version.
|
||||
* @param versions the list of version.
|
||||
*
|
||||
* @return the last stable version.
|
||||
* @param from the reference version (may specify only major, or major+minor, or major+minor+patch).
|
||||
* @param versions the collection of candidate versions to resolve against.
|
||||
* @return the best matching stable version, or {@code null} if none match.
|
||||
*/
|
||||
public static Version getStable(final Version from, final Collection<Version> versions) {
|
||||
List<Version> compatibleVersions = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
if (compatibleVersions.isEmpty()) return null;
|
||||
return Version.getLatest(compatibleVersions);
|
||||
// Case 1: "from" is only a major (e.g. 1)
|
||||
if (from.hasOnlyMajor()) {
|
||||
List<Version> sameMajor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion())
|
||||
.toList();
|
||||
return sameMajor.isEmpty() ? null : Version.getLatest(sameMajor);
|
||||
}
|
||||
|
||||
// Case 2: "from" is major+minor only (e.g. 1.2)
|
||||
if (from.hasMajorAndMinorOnly()) {
|
||||
List<Version> sameMinor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion()
|
||||
&& v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
return sameMinor.isEmpty() ? null : Version.getLatest(sameMinor);
|
||||
}
|
||||
|
||||
// Case 3: "from" is full version (major+minor+patch)
|
||||
if (versions.contains(from)) {
|
||||
return from;
|
||||
}
|
||||
|
||||
// No match
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,8 +159,8 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
private final int majorVersion;
|
||||
private final int minorVersion;
|
||||
private final int incrementalVersion;
|
||||
private final Integer minorVersion;
|
||||
private final Integer patchVersion;
|
||||
private final Qualifier qualifier;
|
||||
|
||||
private final String originalVersion;
|
||||
@@ -134,14 +170,14 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
*/
|
||||
public Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
final int patchVersion,
|
||||
final String qualifier) {
|
||||
this(majorVersion, minorVersion, incrementalVersion, qualifier, null);
|
||||
this(majorVersion, minorVersion, patchVersion, qualifier, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -149,25 +185,25 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
* @param originalVersion the original string version.
|
||||
*/
|
||||
private Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
private Version(final Integer majorVersion,
|
||||
final Integer minorVersion,
|
||||
final Integer patchVersion,
|
||||
final String qualifier,
|
||||
final String originalVersion) {
|
||||
this.majorVersion = requirePositive(majorVersion, "major");
|
||||
this.minorVersion = requirePositive(minorVersion, "minor");
|
||||
this.incrementalVersion = requirePositive(incrementalVersion, "incremental");
|
||||
this.patchVersion = requirePositive(patchVersion, "incremental");
|
||||
this.qualifier = qualifier != null ? new Qualifier(qualifier) : null;
|
||||
this.originalVersion = originalVersion;
|
||||
}
|
||||
|
||||
|
||||
private static int requirePositive(int version, final String message) {
|
||||
if (version < 0) {
|
||||
private static Integer requirePositive(Integer version, final String message) {
|
||||
if (version != null && version < 0) {
|
||||
throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message));
|
||||
}
|
||||
return version;
|
||||
@@ -178,11 +214,11 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
public int minorVersion() {
|
||||
return minorVersion;
|
||||
return minorVersion != null ? minorVersion : 0;
|
||||
}
|
||||
|
||||
public int incrementalVersion() {
|
||||
return incrementalVersion;
|
||||
public int patchVersion() {
|
||||
return patchVersion != null ? patchVersion : 0;
|
||||
}
|
||||
|
||||
public Qualifier qualifier() {
|
||||
@@ -197,9 +233,9 @@ public class Version implements Comparable<Version> {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof Version)) return false;
|
||||
Version version = (Version) o;
|
||||
return majorVersion == version.majorVersion &&
|
||||
minorVersion == version.minorVersion &&
|
||||
incrementalVersion == version.incrementalVersion &&
|
||||
return Objects.equals(majorVersion,version.majorVersion) &&
|
||||
Objects.equals(minorVersion, version.minorVersion) &&
|
||||
Objects.equals(patchVersion,version.patchVersion) &&
|
||||
Objects.equals(qualifier, version.qualifier);
|
||||
}
|
||||
|
||||
@@ -208,7 +244,7 @@ public class Version implements Comparable<Version> {
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier);
|
||||
return Objects.hash(majorVersion, minorVersion, patchVersion, qualifier);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -218,7 +254,7 @@ public class Version implements Comparable<Version> {
|
||||
public String toString() {
|
||||
if (originalVersion != null) return originalVersion;
|
||||
|
||||
String version = majorVersion + "." + minorVersion + "." + incrementalVersion;
|
||||
String version = majorVersion + "." + minorVersion + "." + patchVersion;
|
||||
return (qualifier != null) ? version +"-" + qualifier : version;
|
||||
}
|
||||
|
||||
@@ -238,7 +274,7 @@ public class Version implements Comparable<Version> {
|
||||
return compareMinor;
|
||||
}
|
||||
|
||||
int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion);
|
||||
int compareIncremental = Integer.compare(that.patchVersion, this.patchVersion);
|
||||
if (compareIncremental != 0) {
|
||||
return compareIncremental;
|
||||
}
|
||||
@@ -253,6 +289,21 @@ public class Version implements Comparable<Version> {
|
||||
|
||||
return this.qualifier.compareTo(that.qualifier);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true if only major is specified (e.g. "1")
|
||||
*/
|
||||
private boolean hasOnlyMajor() {
|
||||
return minorVersion == null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if major+minor are specified, but no patch (e.g. "1.2")
|
||||
*/
|
||||
private boolean hasMajorAndMinorOnly() {
|
||||
return minorVersion != null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether this version is before the given one.
|
||||
|
||||
@@ -46,16 +46,19 @@ public class VersionProvider {
|
||||
this.date = loadTime(gitProperties);
|
||||
this.version = loadVersion(buildProperties, gitProperties);
|
||||
|
||||
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
|
||||
if (settingRepository.isPresent()) {
|
||||
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
|
||||
settingRepository.get().save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(this.version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
// check the version in the settings and update if needed, we didn't use it would allow us to detect incompatible update later if needed
|
||||
settingRepository.ifPresent(
|
||||
settingRepositoryInterface -> persistVersion(settingRepositoryInterface, version));
|
||||
}
|
||||
|
||||
private static synchronized void persistVersion(SettingRepositoryInterface settingRepositoryInterface, String version) {
|
||||
Optional<Setting> versionSetting = settingRepositoryInterface.findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(version)) {
|
||||
settingRepositoryInterface.save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@@ -68,6 +69,7 @@ import java.util.Optional;
|
||||
)
|
||||
}
|
||||
)
|
||||
@Slf4j
|
||||
public class Exit extends Task implements ExecutionUpdatableTask {
|
||||
@NotNull
|
||||
@Schema(
|
||||
@@ -104,12 +106,13 @@ public class Exit extends Task implements ExecutionUpdatableTask {
|
||||
// ends all parents
|
||||
while (newTaskRun.getParentTaskRunId() != null) {
|
||||
newTaskRun = newExecution.findTaskRunByTaskRunId(newTaskRun.getParentTaskRunId()).withState(exitState);
|
||||
newExecution = execution.withTaskRun(newTaskRun);
|
||||
newExecution = newExecution.withTaskRun(newTaskRun);
|
||||
}
|
||||
return newExecution;
|
||||
} catch (InternalException e) {
|
||||
// in case we cannot update the last not terminated task run, we ignore it
|
||||
return execution;
|
||||
log.warn("Unable to update the taskrun state", e);
|
||||
return execution.withState(exitState);
|
||||
}
|
||||
})
|
||||
.orElse(execution)
|
||||
|
||||
@@ -127,9 +127,24 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
}
|
||||
|
||||
// check for system labels: none can be passed at runtime
|
||||
Optional<Map.Entry<String, String>> first = labelsAsMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX)).findFirst();
|
||||
if (first.isPresent()) {
|
||||
throw new IllegalArgumentException("System labels can only be set by Kestra itself, offending label: " + first.get().getKey() + "=" + first.get().getValue());
|
||||
Optional<Map.Entry<String, String>> systemLabel = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX))
|
||||
.findFirst();
|
||||
if (systemLabel.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"System labels can only be set by Kestra itself, offending label: " +
|
||||
systemLabel.get().getKey() + "=" + systemLabel.get().getValue()
|
||||
);
|
||||
}
|
||||
|
||||
// check for empty label values
|
||||
Optional<Map.Entry<String, String>> emptyValue = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().isEmpty())
|
||||
.findFirst();
|
||||
if (emptyValue.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label values cannot be empty, offending label: " + emptyValue.get().getKey()
|
||||
);
|
||||
}
|
||||
|
||||
Map<String, String> newLabels = ListUtils.emptyOnNull(execution.getLabels()).stream()
|
||||
@@ -140,6 +155,7 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
newLabels.putAll(labelsAsMap);
|
||||
|
||||
return execution.withLabels(newLabels.entrySet().stream()
|
||||
.filter(Label.getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(
|
||||
entry.getKey(),
|
||||
entry.getValue()
|
||||
|
||||
@@ -216,49 +216,46 @@ public class Subflow extends Task implements ExecutableTask<Subflow.Output>, Chi
|
||||
|
||||
VariablesService variablesService = ((DefaultRunContext) runContext).getApplicationContext().getBean(VariablesService.class);
|
||||
if (this.wait) { // we only compute outputs if we wait for the subflow
|
||||
boolean isOutputsAllowed = runContext
|
||||
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
|
||||
.orElse(true);
|
||||
|
||||
List<io.kestra.core.models.flows.Output> subflowOutputs = flow.getOutputs();
|
||||
|
||||
|
||||
// region [deprecated] Subflow outputs feature
|
||||
if (subflowOutputs == null && isOutputsAllowed && this.getOutputs() != null) {
|
||||
subflowOutputs = this.getOutputs().entrySet().stream()
|
||||
.<io.kestra.core.models.flows.Output>map(entry -> io.kestra.core.models.flows.Output
|
||||
.builder()
|
||||
.id(entry.getKey())
|
||||
.value(entry.getValue())
|
||||
.required(true)
|
||||
.build()
|
||||
)
|
||||
.toList();
|
||||
if (subflowOutputs == null && this.getOutputs() != null) {
|
||||
boolean isOutputsAllowed = runContext
|
||||
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
|
||||
.orElse(true);
|
||||
if (isOutputsAllowed) {
|
||||
try {
|
||||
subflowOutputs = this.getOutputs().entrySet().stream()
|
||||
.<io.kestra.core.models.flows.Output>map(entry -> io.kestra.core.models.flows.Output
|
||||
.builder()
|
||||
.id(entry.getKey())
|
||||
.value(entry.getValue())
|
||||
.required(true)
|
||||
.build()
|
||||
)
|
||||
.toList();
|
||||
} catch (Exception e) {
|
||||
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
|
||||
return failSubflowDueToOutput(runContext, taskRun, execution, e, variables);
|
||||
}
|
||||
} else {
|
||||
runContext.logger().warn("Defining outputs inside the Subflow task is not allowed.");
|
||||
}
|
||||
}
|
||||
//endregion
|
||||
|
||||
if (subflowOutputs != null && !subflowOutputs.isEmpty()) {
|
||||
try {
|
||||
Map<String, Object> outputs = FlowInputOutput.renderFlowOutputs(subflowOutputs, runContext);
|
||||
|
||||
Map<String, Object> rOutputs = FlowInputOutput.renderFlowOutputs(subflowOutputs, runContext);
|
||||
|
||||
FlowInputOutput flowInputOutput = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowInputOutput.class); // this is hacking
|
||||
if (flow.getOutputs() != null && flowInputOutput != null) {
|
||||
outputs = flowInputOutput.typedOutputs(flow, execution, outputs);
|
||||
rOutputs = flowInputOutput.typedOutputs(flow, execution, rOutputs);
|
||||
}
|
||||
builder.outputs(outputs);
|
||||
builder.outputs(rOutputs);
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
|
||||
var state = State.Type.fail(this);
|
||||
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
|
||||
taskRun = taskRun
|
||||
.withState(state)
|
||||
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
|
||||
.withOutputs(variables);
|
||||
|
||||
return Optional.of(SubflowExecutionResult.builder()
|
||||
.executionId(execution.getId())
|
||||
.state(State.Type.FAILED)
|
||||
.parentTaskRun(taskRun)
|
||||
.build());
|
||||
return failSubflowDueToOutput(runContext, taskRun, execution, e, variables);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -282,6 +279,21 @@ public class Subflow extends Task implements ExecutableTask<Subflow.Output>, Chi
|
||||
return Optional.of(ExecutableUtils.subflowExecutionResult(taskRun, execution));
|
||||
}
|
||||
|
||||
private Optional<SubflowExecutionResult> failSubflowDueToOutput(RunContext runContext, TaskRun taskRun, Execution execution, Exception e, Variables outputs) {
|
||||
runContext.logger().error("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
|
||||
var state = State.Type.fail(this);
|
||||
taskRun = taskRun
|
||||
.withState(state)
|
||||
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
|
||||
.withOutputs(outputs);
|
||||
|
||||
return Optional.of(SubflowExecutionResult.builder()
|
||||
.executionId(execution.getId())
|
||||
.state(State.Type.FAILED)
|
||||
.parentTaskRun(taskRun)
|
||||
.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForExecution() {
|
||||
return this.wait;
|
||||
|
||||
@@ -202,7 +202,7 @@ import static io.kestra.core.utils.Rethrow.throwPredicate;
|
||||
code = """
|
||||
id: sentry_execution_example
|
||||
namespace: company.team
|
||||
|
||||
|
||||
tasks:
|
||||
- id: send_alert
|
||||
type: io.kestra.plugin.notifications.sentry.SentryExecution
|
||||
@@ -221,7 +221,7 @@ import static io.kestra.core.utils.Rethrow.throwPredicate;
|
||||
- WARNING
|
||||
- type: io.kestra.plugin.core.condition.ExecutionNamespace
|
||||
namespace: company.payroll
|
||||
prefix: false"""
|
||||
prefix: false"""
|
||||
)
|
||||
|
||||
},
|
||||
@@ -405,6 +405,28 @@ public class Flow extends AbstractTrigger implements TriggerOutput<Flow.Output>
|
||||
return conditions;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public Map<String, Condition> getUpstreamFlowsConditions() {
|
||||
AtomicInteger conditionId = new AtomicInteger();
|
||||
return ListUtils.emptyOnNull(flows).stream()
|
||||
.map(upstreamFlow -> Map.entry(
|
||||
"condition_" + conditionId.incrementAndGet(),
|
||||
new UpstreamFlowCondition(upstreamFlow)
|
||||
))
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public Map<String, Condition> getWhereConditions() {
|
||||
AtomicInteger conditionId = new AtomicInteger();
|
||||
return ListUtils.emptyOnNull(where).stream()
|
||||
.map(filter -> Map.entry(
|
||||
"condition_" + conditionId.incrementAndGet() + "_" + filter.getId(),
|
||||
new FilterCondition(filter)
|
||||
))
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Logger logger() {
|
||||
return log;
|
||||
|
||||
@@ -21,10 +21,13 @@ import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class DocumentationGeneratorTest {
|
||||
@Inject
|
||||
JsonSchemaGenerator jsonSchemaGenerator;
|
||||
|
||||
121
core/src/test/java/io/kestra/core/events/CrudEventTest.java
Normal file
121
core/src/test/java/io/kestra/core/events/CrudEventTest.java
Normal file
@@ -0,0 +1,121 @@
|
||||
package io.kestra.core.events;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
|
||||
|
||||
class CrudEventTest {
|
||||
|
||||
@Test
|
||||
void shouldReturnCreateEventWhenModelIsProvided() {
|
||||
// Given
|
||||
String model = "testModel";
|
||||
|
||||
// When
|
||||
CrudEvent<String> event = CrudEvent.create(model);
|
||||
|
||||
// Then
|
||||
assertThat(event.getModel()).isEqualTo(model);
|
||||
assertThat(event.getPreviousModel()).isNull();
|
||||
assertThat(event.getType()).isEqualTo(CrudEventType.CREATE);
|
||||
assertThat(event.getRequest()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowExceptionWhenCreateEventWithNullModel() {
|
||||
// Given
|
||||
String model = null;
|
||||
|
||||
// When / Then
|
||||
assertThatThrownBy(() -> CrudEvent.create(model))
|
||||
.isInstanceOf(NullPointerException.class)
|
||||
.hasMessage("Can't create CREATE event with a null model");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnDeleteEventWhenModelIsProvided() {
|
||||
// Given
|
||||
String model = "testModel";
|
||||
|
||||
// When
|
||||
CrudEvent<String> event = CrudEvent.delete(model);
|
||||
|
||||
// Then
|
||||
assertThat(event.getModel()).isNull();
|
||||
assertThat(event.getPreviousModel()).isEqualTo(model);
|
||||
assertThat(event.getType()).isEqualTo(CrudEventType.DELETE);
|
||||
assertThat(event.getRequest()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowExceptionWhenDeleteEventWithNullModel() {
|
||||
// Given
|
||||
String model = null;
|
||||
|
||||
// When / Then
|
||||
assertThatThrownBy(() -> CrudEvent.delete(model))
|
||||
.isInstanceOf(NullPointerException.class)
|
||||
.hasMessage("Can't create DELETE event with a null model");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnUpdateEventWhenBeforeAndAfterAreProvided() {
|
||||
// Given
|
||||
String before = "oldModel";
|
||||
String after = "newModel";
|
||||
|
||||
// When
|
||||
CrudEvent<String> event = CrudEvent.of(before, after);
|
||||
|
||||
// Then
|
||||
assertThat(event.getModel()).isEqualTo(after);
|
||||
assertThat(event.getPreviousModel()).isEqualTo(before);
|
||||
assertThat(event.getType()).isEqualTo(CrudEventType.UPDATE);
|
||||
assertThat(event.getRequest()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnCreateEventWhenBeforeIsNullAndAfterIsProvided() {
|
||||
// Given
|
||||
String before = null;
|
||||
String after = "newModel";
|
||||
|
||||
// When
|
||||
CrudEvent<String> event = CrudEvent.of(before, after);
|
||||
|
||||
// Then
|
||||
assertThat(event.getModel()).isEqualTo(after);
|
||||
assertThat(event.getPreviousModel()).isNull();
|
||||
assertThat(event.getType()).isEqualTo(CrudEventType.CREATE);
|
||||
assertThat(event.getRequest()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnDeleteEventWhenAfterIsNullAndBeforeIsProvided() {
|
||||
// Given
|
||||
String before = "oldModel";
|
||||
String after = null;
|
||||
|
||||
// When
|
||||
CrudEvent<String> event = CrudEvent.of(before, after);
|
||||
|
||||
// Then
|
||||
assertThat(event.getModel()).isNull();
|
||||
assertThat(event.getPreviousModel()).isEqualTo(before);
|
||||
assertThat(event.getType()).isEqualTo(CrudEventType.DELETE);
|
||||
assertThat(event.getRequest()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowExceptionWhenBothBeforeAndAfterAreNull() {
|
||||
// Given
|
||||
String before = null;
|
||||
String after = null;
|
||||
|
||||
// When / Then
|
||||
assertThatThrownBy(() -> CrudEvent.of(before, after))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessage("Both before and after cannot be null");
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ import lombok.Value;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
@@ -67,6 +68,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@Testcontainers
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
class HttpClientTest {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
@@ -1,19 +1,32 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.validations.ModelValidator;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class LabelTest {
|
||||
@Inject
|
||||
private ModelValidator modelValidator;
|
||||
|
||||
@Test
|
||||
void shouldGetNestedMapGivenDistinctLabels() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label(Label.USERNAME, "test"),
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
new Label(Label.CORRELATION_ID, "id"),
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null)
|
||||
)
|
||||
);
|
||||
|
||||
assertThat(result).isEqualTo(
|
||||
@@ -34,6 +47,18 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toNestedMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGetMapGivenDistinctLabels() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
@@ -59,6 +84,18 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDuplicateLabelsWithKeyOrderKept() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
@@ -73,4 +110,28 @@ class LabelTest {
|
||||
new Label(Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deduplicateShouldIgnoreEmptyAndNull() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldValidateEmpty() {
|
||||
Optional<ConstraintViolationException> validLabelResult = modelValidator.isValid(new Label("foo", "bar"));
|
||||
assertThat(validLabelResult.isPresent()).isFalse();
|
||||
|
||||
Optional<ConstraintViolationException> emptyValueLabelResult = modelValidator.isValid(new Label("foo", ""));
|
||||
assertThat(emptyValueLabelResult.isPresent()).isTrue();
|
||||
|
||||
Optional<ConstraintViolationException> emptyKeyLabelResult = modelValidator.isValid(new Label("", "bar"));
|
||||
assertThat(emptyKeyLabelResult.isPresent()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -13,19 +13,19 @@ import java.util.Map;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ExecutionTest {
|
||||
private static final TaskRun.TaskRunBuilder TASK_RUN = TaskRun.builder()
|
||||
.id("test");
|
||||
|
||||
@Test
|
||||
void hasTaskRunJoinableTrue() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -36,13 +36,15 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableSameState() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State())
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State())
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -51,7 +53,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableFailedExecutionFromExecutor() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -59,7 +62,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -68,7 +72,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartFailed() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -77,7 +82,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -88,7 +94,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartSuccess() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -97,7 +104,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -109,7 +117,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableAfterRestart() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -118,7 +127,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
|
||||
@@ -7,12 +7,11 @@ import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.TriggerRepositoryInterface;
|
||||
import io.kestra.core.runners.RunnerUtils;
|
||||
import io.kestra.core.runners.TestRunnerUtils;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
import io.kestra.core.services.GraphService;
|
||||
import io.kestra.core.utils.GraphUtils;
|
||||
@@ -45,7 +44,7 @@ class FlowGraphTest {
|
||||
private TriggerRepositoryInterface triggerRepositoryInterface;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
@Test
|
||||
void simple() throws IllegalVariableEvaluationException, IOException {
|
||||
@@ -261,10 +260,10 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant1")
|
||||
void subflow() throws IllegalVariableEvaluationException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml");
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml", "tenant1");
|
||||
FlowGraph flowGraph = GraphUtils.flowGraph(flow, null);
|
||||
|
||||
assertThat(flowGraph.getNodes().size()).isEqualTo(6);
|
||||
@@ -293,15 +292,15 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant2")
|
||||
void dynamicIdSubflow() throws IllegalVariableEvaluationException, TimeoutException, QueueException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml").toBuilder().revision(1).build();
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml", "tenant2").toBuilder().revision(1).build();
|
||||
|
||||
IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> graphService.flowGraph(flow, Collections.singletonList("root.launch")));
|
||||
assertThat(illegalArgumentException.getMessage()).isEqualTo("Can't expand subflow task 'launch' because namespace and/or flowId contains dynamic values. This can only be viewed on an execution.");
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
Execution execution = runnerUtils.runOne("tenant2", "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
"namespace", f.getNamespace(),
|
||||
"flowId", "switch"
|
||||
));
|
||||
@@ -373,13 +372,17 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path) throws IOException {
|
||||
return parse(path, MAIN_TENANT);
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path, String tenantId) throws IOException {
|
||||
URL resource = TestsUtils.class.getClassLoader().getResource(path);
|
||||
assert resource != null;
|
||||
|
||||
File file = new File(resource.getFile());
|
||||
|
||||
return YamlParser.parse(file, FlowWithSource.class).toBuilder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.source(Files.readString(file.toPath()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.runners.*;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
@@ -84,8 +85,9 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfile() throws IOException {
|
||||
URI uri = createNsFile(false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "namespace")));
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", namespace)));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
String fetchedContent = new String(fetch.readAllBytes());
|
||||
@@ -95,7 +97,8 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfileFromOtherNs() throws IOException {
|
||||
URI uri = createNsFile(true);
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, true);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "other")));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
@@ -139,8 +142,7 @@ class URIFetcherTest {
|
||||
);
|
||||
}
|
||||
|
||||
private URI createNsFile(boolean nsInAuthority) throws IOException {
|
||||
String namespace = "namespace";
|
||||
private URI createNsFile(String namespace, boolean nsInAuthority) throws IOException {
|
||||
String filePath = "file.txt";
|
||||
storage.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storage.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
|
||||
@@ -10,6 +10,7 @@ import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -33,17 +34,15 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorage() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
var command = ScriptService.replaceInternalStorage(runContext, null, false);
|
||||
assertThat(command).isEqualTo("");
|
||||
|
||||
command = ScriptService.replaceInternalStorage(runContext, "my command", false);
|
||||
assertThat(command).isEqualTo("my command");
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
File localFile = null;
|
||||
@@ -70,12 +69,10 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorageUnicode() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file-龍.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file-龍");
|
||||
|
||||
String internalStorageUri = "kestra://some/file-龍.txt";
|
||||
File localFile = null;
|
||||
@@ -95,12 +92,10 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadInputFiles() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
List<File> filesToDelete = new ArrayList<>();
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
@@ -143,13 +138,11 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadOutputFiles() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/main"));
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
assertThat(outputFiles, not(anEmptyMap()));
|
||||
assertThat(outputFiles.get("file.txt")).isEqualTo(URI.create("kestra:///file.txt"));
|
||||
|
||||
@@ -232,4 +225,13 @@ class ScriptServiceTest {
|
||||
.build();
|
||||
return runContextFactory.of(flow, task, execution, taskRun);
|
||||
}
|
||||
|
||||
private static Path createFile(String tenant, String fileName) throws IOException {
|
||||
Path path = Path.of("/tmp/unittest/%s/%s.txt".formatted(tenant, fileName));
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createDirectory(Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
Files.createFile(path);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package io.kestra.core.models.triggers.multipleflows;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import io.kestra.plugin.core.condition.ExecutionFlow;
|
||||
@@ -33,8 +34,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void allDefault() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -50,8 +52,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void daily() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -67,8 +70,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void dailyAdvance() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -84,8 +88,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void hourly() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -102,8 +107,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void minutely() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -115,8 +121,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expiration() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -136,8 +143,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -146,20 +154,21 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -168,20 +177,21 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline_Expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -190,16 +200,17 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults()).isEmpty();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeWindow() throws Exception {
|
||||
void dailyTimeWindow() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
LocalTime startTime = LocalTime.now().truncatedTo(ChronoUnit.MINUTES);
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -208,15 +219,16 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void slidingWindow() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -225,13 +237,13 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(TimeWindow sla) {
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(String tenantId, TimeWindow sla) {
|
||||
var multipleCondition = MultipleCondition.builder()
|
||||
.id("condition-multiple")
|
||||
.id("condition-multiple-%s".formatted(tenantId))
|
||||
.conditions(ImmutableMap.of(
|
||||
"flow-a", ExecutionFlow.builder()
|
||||
.flowId(Property.ofValue("flow-a"))
|
||||
@@ -248,6 +260,7 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
Flow flow = Flow.builder()
|
||||
.namespace(NAMESPACE)
|
||||
.id("multiple-flow")
|
||||
.tenantId(tenantId)
|
||||
.revision(1)
|
||||
.triggers(Collections.singletonList(io.kestra.plugin.core.trigger.Flow.builder()
|
||||
.id("trigger-flow")
|
||||
|
||||
@@ -13,21 +13,20 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
public abstract class AbstractFeatureUsageReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
FeatureUsageReport featureUsageReport;
|
||||
|
||||
|
||||
@Test
|
||||
public void shouldGetReport() {
|
||||
// When
|
||||
Instant now = Instant.now();
|
||||
FeatureUsageReport.UsageEvent event = featureUsageReport.report(
|
||||
now,
|
||||
now,
|
||||
Reportable.TimeInterval.of(now.minus(Duration.ofDays(1)).atZone(ZoneId.systemDefault()), now.atZone(ZoneId.systemDefault()))
|
||||
);
|
||||
|
||||
|
||||
// Then
|
||||
assertThat(event.getExecutions().getDailyExecutionsCount().size()).isGreaterThan(0);
|
||||
assertThat(event.getExecutions().getDailyTaskRunsCount()).isNull();
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class SystemInformationReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
private SystemInformationReport systemInformationReport;
|
||||
|
||||
|
||||
@Test
|
||||
void shouldGetReport() {
|
||||
SystemInformationReport.SystemInformationEvent event = systemInformationReport.report(Instant.now());
|
||||
@@ -32,34 +32,34 @@ class SystemInformationReportTest {
|
||||
assertThat(event.host().getHardware().getLogicalProcessorCount()).isNotNull();
|
||||
assertThat(event.host().getJvm().getName()).isNotNull();
|
||||
assertThat(event.host().getOs().getFamily()).isNotNull();
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("memory");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("memory");
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("h2");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("h2");
|
||||
}
|
||||
|
||||
|
||||
@MockBean(SettingRepositoryInterface.class)
|
||||
@Singleton
|
||||
static class TestSettingRepository implements SettingRepositoryInterface {
|
||||
public static Object UUID = null;
|
||||
|
||||
|
||||
@Override
|
||||
public Optional<Setting> findByKey(String key) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Setting> findAll() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting save(Setting setting) throws ConstraintViolationException {
|
||||
if (setting.getKey().equals(Setting.INSTANCE_UUID)) {
|
||||
UUID = setting.getValue();
|
||||
}
|
||||
|
||||
|
||||
return setting;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting delete(Setting setting) {
|
||||
return setting;
|
||||
|
||||
@@ -25,6 +25,7 @@ import io.kestra.core.models.tasks.ResolvedTask;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.NamespaceUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
@@ -48,7 +49,6 @@ import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
@@ -62,17 +62,17 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId) {
|
||||
return builder(state, flowId, NAMESPACE);
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId) {
|
||||
return builder(tenantId, state, flowId, NAMESPACE);
|
||||
}
|
||||
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId, String namespace) {
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId, String namespace) {
|
||||
State finalState = randomDuration(state);
|
||||
|
||||
Execution.ExecutionBuilder execution = Execution.builder()
|
||||
.id(FriendlyId.createFriendlyId())
|
||||
.namespace(namespace)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.flowId(flowId == null ? FLOW : flowId)
|
||||
.flowRevision(1)
|
||||
.state(finalState);
|
||||
@@ -126,11 +126,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
return finalState;
|
||||
}
|
||||
|
||||
protected void inject() {
|
||||
inject(null);
|
||||
protected void inject(String tenantId) {
|
||||
inject(tenantId, null);
|
||||
}
|
||||
|
||||
protected void inject(String executionTriggerId) {
|
||||
protected void inject(String tenantId, String executionTriggerId) {
|
||||
ExecutionTrigger executionTrigger = null;
|
||||
|
||||
if (executionTriggerId != null) {
|
||||
@@ -139,7 +139,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
executionRepository.save(builder(State.Type.RUNNING, null)
|
||||
executionRepository.save(builder(tenantId, State.Type.RUNNING, null)
|
||||
.labels(List.of(
|
||||
new Label("key", "value"),
|
||||
new Label("key2", "value2")
|
||||
@@ -149,6 +149,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
);
|
||||
for (int i = 1; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).trigger(executionTrigger).build());
|
||||
@@ -156,6 +157,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
// add a test execution, this should be ignored in search & statistics
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
State.Type.SUCCESS,
|
||||
null
|
||||
)
|
||||
@@ -167,9 +169,10 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter, int expectedSize){
|
||||
inject("executionTriggerId");
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, "executionTriggerId");
|
||||
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(expectedSize);
|
||||
}
|
||||
@@ -192,7 +195,8 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -208,9 +212,10 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void find() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -219,7 +224,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value( List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -227,7 +232,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(1L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -235,7 +240,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value2"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -244,7 +249,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(Map.of("key", "value", "keyTest", "valueTest"))
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -252,7 +257,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value("second")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -266,7 +271,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(NAMESPACE)
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -274,7 +279,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.STARTS_WITH)
|
||||
.value("io.kestra")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
}
|
||||
|
||||
@@ -282,15 +287,16 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
protected void findTriggerExecutionId() {
|
||||
String executionTriggerId = IdUtils.create();
|
||||
|
||||
inject(executionTriggerId);
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, executionTriggerId);
|
||||
inject(tenant);
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.TRIGGER_EXECUTION_ID)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(executionTriggerId)
|
||||
.build());
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -300,7 +306,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.CHILD)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -311,20 +317,21 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.MAIN)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters );
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters );
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger()).isNull();
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(56L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findWithSort() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), MAIN_TENANT, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -333,100 +340,92 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findTaskRun() {
|
||||
inject();
|
||||
|
||||
ArrayListTotal<TaskRun> taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(74L);
|
||||
assertThat(taskRuns.size()).isEqualTo(10);
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
|
||||
taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(1L);
|
||||
assertThat(taskRuns.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
protected void findById() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(execution1.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByIdTestExecution() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_TEST);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var executionTest = ExecutionFixture.EXECUTION_TEST(tenant);
|
||||
executionRepository.save(executionTest);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(null, ExecutionFixture.EXECUTION_TEST.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, executionTest.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_TEST.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(executionTest.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void purge() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.purge(ExecutionFixture.EXECUTION_1);
|
||||
executionRepository.purge(execution1);
|
||||
|
||||
full = executionRepository.findById(null, ExecutionFixture.EXECUTION_1.getId());
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void delete() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.delete(ExecutionFixture.EXECUTION_1);
|
||||
executionRepository.delete(execution1);
|
||||
|
||||
full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void mappingConflict() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2(tenant));
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1(tenant));
|
||||
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(MAIN_TENANT, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(tenant, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
|
||||
assertThat(page1.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void dailyStatistics() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
@@ -436,15 +435,14 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
@@ -456,131 +454,52 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(21L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(20L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void taskRunsDailyStatistics() {
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getDuration().getAvg().toMillis()).isGreaterThan(0L);
|
||||
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED)).isEqualTo(3L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.RUNNING)).isEqualTo(5L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(55L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@SuppressWarnings("OptionalGetWithoutIsPresent")
|
||||
@Test
|
||||
protected void executionsCount() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 14; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
i < 2 ? "first" : (i < 5 ? "second" : "third")
|
||||
).build());
|
||||
@@ -590,7 +509,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
Thread.sleep(500);
|
||||
|
||||
List<ExecutionCount> result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -609,7 +528,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("missing")).findFirst().get().getCount()).isEqualTo(0L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -626,7 +545,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("third")).findFirst().get().getCount()).isEqualTo(9L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
@@ -639,14 +558,15 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void update() {
|
||||
Execution execution = ExecutionFixture.EXECUTION_1;
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution execution = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution);
|
||||
|
||||
Label label = new Label("key", "value");
|
||||
Execution updated = execution.toBuilder().labels(List.of(label)).build();
|
||||
executionRepository.update(updated);
|
||||
|
||||
Optional<Execution> validation = executionRepository.findById(MAIN_TENANT, updated.getId());
|
||||
Optional<Execution> validation = executionRepository.findById(tenant, updated.getId());
|
||||
assertThat(validation.isPresent()).isTrue();
|
||||
assertThat(validation.get().getLabels().size()).isEqualTo(1);
|
||||
assertThat(validation.get().getLabels().getFirst()).isEqualTo(label);
|
||||
@@ -654,13 +574,14 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldFindLatestExecutionGivenState() {
|
||||
Execution earliest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(5)));
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution earliest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(5)));
|
||||
|
||||
executionRepository.save(earliest);
|
||||
executionRepository.save(latest);
|
||||
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(MAIN_TENANT, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(tenant, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
assertThat(result.isPresent()).isTrue();
|
||||
assertThat(result.get().getId()).isEqualTo(latest.getId());
|
||||
}
|
||||
@@ -700,11 +621,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(data.get(0).get("date")).isEqualTo(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").format(ZonedDateTime.ofInstant(startDate, ZoneId.systemDefault()).withSecond(0).withNano(0)));
|
||||
}
|
||||
|
||||
private static Execution buildWithCreatedDate(Instant instant) {
|
||||
private static Execution buildWithCreatedDate(String tenant, Instant instant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State(State.Type.CREATED, List.of(new State.History(State.Type.CREATED, instant))))
|
||||
@@ -715,22 +636,24 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void findAllAsync() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<Execution> executions = executionRepository.findAllAsync(MAIN_TENANT).collectList().block();
|
||||
List<Execution> executions = executionRepository.findAllAsync(tenant).collectList().block();
|
||||
assertThat(executions).hasSize(29); // used by the backup so it contains TEST executions
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByLabel() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<QueryFilter> filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.size()).isEqualTo(1L);
|
||||
|
||||
// Filtering by two pairs of labels, since now its a and behavior, it should not return anything
|
||||
@@ -739,15 +662,16 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value", "keyother", "valueother"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.size()).isEqualTo(0L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastExecutionsWhenInputsAreNull() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(MAIN_TENANT, null);
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(tenant, null);
|
||||
|
||||
assertThat(lastExecutions).isNotEmpty();
|
||||
Set<String> flowIds = lastExecutions.stream().map(Execution::getFlowId).collect(Collectors.toSet());
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.Helpers;
|
||||
import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.exceptions.InvalidQueryFiltersException;
|
||||
@@ -10,7 +9,6 @@ import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.QueryFilter.Field;
|
||||
import io.kestra.core.models.QueryFilter.Op;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.conditions.ConditionContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.ExecutionTrigger;
|
||||
@@ -20,7 +18,6 @@ import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.models.triggers.PollingTriggerInterface;
|
||||
import io.kestra.core.models.triggers.TriggerContext;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.services.FlowService;
|
||||
import io.kestra.core.utils.Await;
|
||||
@@ -29,22 +26,19 @@ import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.slf4j.event.Level;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
@@ -52,16 +46,12 @@ import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
// If some counts are wrong in this test it means that one of the tests is not properly deleting what it created
|
||||
@KestraTest
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public abstract class AbstractFlowRepositoryTest {
|
||||
public static final String TEST_TENANT_ID = "tenant";
|
||||
public static final String TEST_NAMESPACE = "io.kestra.unittest";
|
||||
public static final String TEST_FLOW_ID = "test";
|
||||
@Inject
|
||||
@@ -70,21 +60,18 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@Inject
|
||||
private LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
TestsUtils.loads(MAIN_TENANT, repositoryLoader);
|
||||
@BeforeAll
|
||||
protected static void init() {
|
||||
FlowListener.reset();
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder() {
|
||||
return builder(IdUtils.create(), TEST_FLOW_ID);
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, IdUtils.create(), TEST_FLOW_ID);
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String flowId, String taskId) {
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId, String flowId, String taskId) {
|
||||
return FlowWithSource.builder()
|
||||
.tenantId(tenantId)
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tasks(Collections.singletonList(Return.builder().id(taskId).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()));
|
||||
@@ -93,16 +80,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -113,16 +100,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_with_source(QueryFilter filter){
|
||||
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -144,7 +131,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> flowRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -153,7 +140,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all_with_source(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -176,17 +163,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
Optional<Flow> full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -195,17 +182,18 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithoutAcl() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -214,15 +202,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithSource() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
String source = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, source));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, source));
|
||||
|
||||
try {
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
@@ -237,7 +226,8 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
FlowWithSource flow = builder().revision(12).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).revision(12).build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -249,7 +239,8 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void saveNoRevision() {
|
||||
FlowWithSource flow = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -260,68 +251,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
List<Flow> save = flowRepository.findAll(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSource(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
List<Flow> save = flowRepository.findAllForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSourceForAllTenants() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSourceForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
List<Flow> save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 24);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefix() {
|
||||
List<Flow> save = flowRepository.findByNamespacePrefix(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespaceWithSource() {
|
||||
Flow flow = builder()
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
String flowSource = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, flowSource));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, flowSource));
|
||||
|
||||
try {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(MAIN_TENANT, flow.getNamespace());
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(tenant, flow.getNamespace());
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
assertThat(save.getFirst().getSource()).isEqualTo(FlowService.cleanupSource(flowSource));
|
||||
@@ -330,175 +270,15 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefixWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespacePrefixWithSource(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationPartial() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationGreaterThanExisting() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_prefixMatchingAllNamespaces() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecifiedNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificSubNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificFlowByNamespaceAndLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_noResult_forAnUnknownNamespace() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findSpecialChars() {
|
||||
ArrayListTotal<SearchResult<Flow>> save = flowRepository.findSourceCode(Pageable.unpaged(), "https://api.chucknorris.io", MAIN_TENANT, null);
|
||||
assertThat((long) save.size()).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void delete() {
|
||||
Flow flow = builder().tenantId(MAIN_TENANT).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant).tenantId(tenant).build();
|
||||
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
} catch (Throwable e) {
|
||||
deleteFlow(save);
|
||||
throw e;
|
||||
@@ -506,21 +286,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow delete = flowRepository.delete(save);
|
||||
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(revisions.getLast().getRevision()).isEqualTo(delete.getRevision());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateConflict() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("a").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -528,12 +309,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest2")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("b").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -551,13 +332,14 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void removeTrigger() throws TimeoutException, QueueException {
|
||||
public void removeTrigger() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -567,12 +349,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
;
|
||||
@@ -583,21 +365,25 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
deleteFlow(flow);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.getEmits().size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void removeTriggerDelete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -607,40 +393,39 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(save);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.getEmits().size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findDistinctNamespace() {
|
||||
List<String> distinctNamespace = flowRepository.findDistinctNamespace(MAIN_TENANT);
|
||||
assertThat((long) distinctNamespace.size()).isEqualTo(9L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnNullRevisionForNonExistingFlow() {
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
assertThat(flowRepository.lastRevision(TestsUtils.randomTenant(this.getClass().getSimpleName()), TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastRevisionOnCreate() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId);
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId);
|
||||
|
||||
// Then
|
||||
assertThat(result).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -649,34 +434,36 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final String flowId = IdUtils.create();
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
|
||||
// When
|
||||
flowRepository.delete(created);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnCreateAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
flowRepository.delete(
|
||||
flowRepository.create(createTestingLogFlow(flowId, "first"))
|
||||
flowRepository.create(createTestingLogFlow(tenant, flowId, "first"))
|
||||
);
|
||||
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "second")));
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "second")));
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -685,22 +472,23 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldReturnNullForLastRevisionAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isNull();
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isNull();
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -709,22 +497,23 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldFindAllRevisionsAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -732,21 +521,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnUpdateGivenNotEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
@@ -755,48 +545,39 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldNotIncrementRevisionOnUpdateGivenEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "first"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "first"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnForGivenQueryWildCardFilters() {
|
||||
List<QueryFilter> filters = List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.QUERY).operation(QueryFilter.Op.EQUALS).value("*").build()
|
||||
);
|
||||
ArrayListTotal<Flow> flows = flowRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(flows.size()).isEqualTo(10);
|
||||
assertThat(flows.getTotal()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByExecution() {
|
||||
Flow flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(1)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.flowId(flow.getId())
|
||||
.flowRevision(flow.getRevision())
|
||||
.state(new State())
|
||||
@@ -821,11 +602,13 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByExecutionNoRevision() {
|
||||
Flow flow = builder()
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.tenantId(tenant)
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.flowId(flow.getId())
|
||||
@@ -851,13 +634,14 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource toDelete = null;
|
||||
try {
|
||||
// Given
|
||||
Flow flow = createTestFlowForNamespace(TEST_NAMESPACE);
|
||||
Flow flow = createTestFlowForNamespace(tenant, TEST_NAMESPACE);
|
||||
toDelete = flowRepository.create(GenericFlow.of(flow));
|
||||
// When
|
||||
int count = flowRepository.count(MAIN_TENANT);
|
||||
int count = flowRepository.count(tenant);
|
||||
|
||||
// Then
|
||||
Assertions.assertTrue(count > 0);
|
||||
@@ -868,11 +652,11 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static Flow createTestFlowForNamespace(String namespace) {
|
||||
private static Flow createTestFlowForNamespace(String tenantId, String namespace) {
|
||||
return Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.tasks(List.of(Return.builder()
|
||||
.id(IdUtils.create())
|
||||
.type(Return.class.getName())
|
||||
@@ -891,21 +675,31 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
|
||||
@Getter
|
||||
private static List<CrudEvent<Flow>> emits = new ArrayList<>();
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<AbstractFlow>> {
|
||||
private static List<CrudEvent<AbstractFlow>> emits = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<Flow> event) {
|
||||
emits.add(event);
|
||||
public void onApplicationEvent(CrudEvent<AbstractFlow> event) {
|
||||
//This has to be done because Micronaut may send CrudEvent<Setting> for example, and we don't want them.
|
||||
if ((event.getModel() != null && event.getModel() instanceof AbstractFlow)||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof AbstractFlow)) {
|
||||
emits.add(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
emits = new ArrayList<>();
|
||||
emits = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
public static List<CrudEvent<AbstractFlow>> filterByTenant(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getModel() != null && e.getModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
private static GenericFlow createTestingLogFlow(String id, String logMessage) {
|
||||
private static GenericFlow createTestingLogFlow(String tenantId, String id, String logMessage) {
|
||||
String source = """
|
||||
id: %s
|
||||
namespace: %s
|
||||
@@ -914,7 +708,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: %s
|
||||
""".formatted(id, TEST_NAMESPACE, logMessage);
|
||||
return GenericFlow.fromYaml(TEST_TENANT_ID, source);
|
||||
return GenericFlow.fromYaml(tenantId, source);
|
||||
}
|
||||
|
||||
protected static int COUNTER = 0;
|
||||
|
||||
@@ -4,7 +4,7 @@ import io.kestra.core.models.topologies.FlowNode;
|
||||
import io.kestra.core.models.topologies.FlowRelation;
|
||||
import io.kestra.core.models.topologies.FlowTopology;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.tenant.TenantService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -17,21 +17,21 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
@Inject
|
||||
private FlowTopologyRepositoryInterface flowTopologyRepository;
|
||||
|
||||
protected FlowTopology createSimpleFlowTopology(String flowA, String flowB, String namespace) {
|
||||
protected FlowTopology createSimpleFlowTopology(String tenantId, String flowA, String flowB, String namespace) {
|
||||
return FlowTopology.builder()
|
||||
.relation(FlowRelation.FLOW_TASK)
|
||||
.source(FlowNode.builder()
|
||||
.id(flowA)
|
||||
.namespace(namespace)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowA)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowA)
|
||||
.build()
|
||||
)
|
||||
.destination(FlowNode.builder()
|
||||
.id(flowB)
|
||||
.namespace(namespace)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowB)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowB)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
@@ -39,42 +39,45 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByFlow() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(TenantService.MAIN_TENANT, "io.kestra.tests", "flow-a", false);
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(tenant, "io.kestra.tests", "flow-a", false);
|
||||
|
||||
assertThat(list.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(TenantService.MAIN_TENANT, "io.kestra.tests");
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(tenant, "io.kestra.tests");
|
||||
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-e", "flow-f", "io.kestra.tests.2")
|
||||
createSimpleFlowTopology(tenant, "flow-e", "flow-f", "io.kestra.tests.2")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(TenantService.MAIN_TENANT);
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(tenant);
|
||||
|
||||
assertThat(list.size()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import io.kestra.core.models.executions.LogEntry;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Logs;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -32,9 +33,7 @@ import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatReflectiveOperationException;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@@ -42,11 +41,11 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@Inject
|
||||
protected LogRepositoryInterface logRepository;
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level) {
|
||||
return logEntry(level, IdUtils.create());
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level) {
|
||||
return logEntry(tenantId, level, IdUtils.create());
|
||||
}
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level, String executionId) {
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level, String executionId) {
|
||||
return LogEntry.builder()
|
||||
.flowId("flowId")
|
||||
.namespace("io.kestra.unittest")
|
||||
@@ -57,7 +56,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
.timestamp(Instant.now())
|
||||
.level(level)
|
||||
.thread("")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.triggerId("triggerId")
|
||||
.message("john doe");
|
||||
}
|
||||
@@ -65,9 +64,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -75,9 +75,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_async(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAsync(MAIN_TENANT, List.of(filter));
|
||||
Flux<LogEntry> find = logRepository.findAsync(tenant, List.of(filter));
|
||||
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(1);
|
||||
@@ -86,11 +87,12 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_delete_with_filter(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
logRepository.deleteByFilters(MAIN_TENANT, List.of(filter));
|
||||
logRepository.deleteByFilters(tenant, List.of(filter));
|
||||
|
||||
assertThat(logRepository.findAllAsync(MAIN_TENANT).collectList().block()).isEmpty();
|
||||
assertThat(logRepository.findAllAsync(tenant).collectList().block()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
@@ -150,7 +152,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> logRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
TestsUtils.randomTenant(this.getClass().getSimpleName()),
|
||||
List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -168,16 +173,17 @@ public abstract class AbstractLogRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
|
||||
LogEntry save = logRepository.save(builder.build());
|
||||
logRepository.save(builder.executionKind(ExecutionKind.TEST).build()); // should only be loaded by execution id
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
@@ -193,7 +199,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
find = logRepository.find(Pageable.UNPAGED, "doe", filters);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
@@ -201,141 +207,146 @@ public abstract class AbstractLogRepositoryTest {
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
List<LogEntry> list = logRepository.findByExecutionId(MAIN_TENANT, save.getExecutionId(), null);
|
||||
List<LogEntry> list = logRepository.findByExecutionId(tenant, save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
list = logRepository.findByExecutionId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(tenant, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
Integer countDeleted = logRepository.purge(Execution.builder().id(save.getExecutionId()).build());
|
||||
assertThat(countDeleted).isEqualTo(2);
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageable() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = "123";
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
builder.executionId(executionId);
|
||||
|
||||
for (int i = 0; i < 80; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
builder = logEntry(Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
builder = logEntry(tenant, Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
LogEntry logEntry2 = logRepository.save(builder.build());
|
||||
for (int i = 0; i < 20; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(50);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(3, 50));
|
||||
find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(3, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionIdAndTaskId(tenant, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(21);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindByExecutionIdTestLogs() {
|
||||
var builder = logEntry(Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var builder = logEntry(tenant, Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
logRepository.save(builder);
|
||||
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(MAIN_TENANT, builder.getExecutionId(), null);
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(tenant, builder.getExecutionId(), null);
|
||||
assertThat(logs).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteByQuery() {
|
||||
LogEntry log1 = logEntry(Level.INFO).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry log1 = logEntry(tenant, Level.INFO).build();
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, log1.getExecutionId(), null, null, null, null);
|
||||
logRepository.deleteByQuery(tenant, log1.getExecutionId(), null, null, null, null);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null);
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null);
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(tenant, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
logRepository.save(logEntry(Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(Level.ERROR).build());
|
||||
logRepository.save(logEntry(Level.WARN).build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(tenant, Level.ERROR).build());
|
||||
logRepository.save(logEntry(tenant, Level.WARN).build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(MAIN_TENANT);
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(tenant);
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fetchData() throws IOException {
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
|
||||
var results = logRepository.fetchData(MAIN_TENANT,
|
||||
var results = logRepository.fetchData(tenant,
|
||||
Logs.builder()
|
||||
.type(Logs.class.getName())
|
||||
.columns(Map.of(
|
||||
|
||||
@@ -7,6 +7,7 @@ import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.executions.metrics.Counter;
|
||||
import io.kestra.core.models.executions.metrics.MetricAggregations;
|
||||
import io.kestra.core.models.executions.metrics.Timer;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -25,27 +26,28 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
MetricEntry testCounter = MetricEntry.of(taskRun1, counter("test"), ExecutionKind.TEST);
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(testCounter); // should only be retrieved by execution id
|
||||
metricRepository.save(timer);
|
||||
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(null, executionId, Pageable.from(1, 10));
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(tenant, executionId, Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskId(null, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskId(tenant, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(null, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(tenant, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(2);
|
||||
|
||||
MetricAggregations aggregationResults = metricRepository.aggregateByFlowId(
|
||||
null,
|
||||
tenant,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -59,7 +61,7 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
assertThat(aggregationResults.getGroupBy()).isEqualTo("day");
|
||||
|
||||
aggregationResults = metricRepository.aggregateByFlowId(
|
||||
null,
|
||||
tenant,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -76,11 +78,12 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void names() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
|
||||
TaskRun taskRun2 = taskRun(executionId, "task2");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task2");
|
||||
MetricEntry counter2 = MetricEntry.of(taskRun2, counter("counter2"), null);
|
||||
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
@@ -90,9 +93,9 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
metricRepository.save(test); // should only be retrieved by execution id
|
||||
|
||||
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(null, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(null, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(null, "namespace", "flow");
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(tenant, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(tenant, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(tenant, "namespace", "flow");
|
||||
|
||||
assertThat(flowMetricsNames.size()).isEqualTo(2);
|
||||
assertThat(taskMetricsNames.size()).isEqualTo(1);
|
||||
@@ -101,17 +104,18 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(timer);
|
||||
metricRepository.save(test); // should be retrieved as findAllAsync is used for backup
|
||||
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(null).collectList().block();
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(tenant).collectList().block();
|
||||
assertThat(results).hasSize(3);
|
||||
}
|
||||
|
||||
@@ -123,8 +127,9 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
return Timer.of("counter", Duration.ofSeconds(5));
|
||||
}
|
||||
|
||||
private TaskRun taskRun(String executionId, String taskId) {
|
||||
private TaskRun taskRun(String tenantId, String executionId, String taskId) {
|
||||
return TaskRun.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId("flow")
|
||||
.namespace("namespace")
|
||||
.executionId(executionId)
|
||||
|
||||
@@ -4,6 +4,8 @@ import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.templates.Template;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
@@ -11,7 +13,10 @@ import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -20,6 +25,8 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -28,55 +35,60 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
@Inject
|
||||
protected TemplateRepositoryInterface templateRepository;
|
||||
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
@BeforeAll
|
||||
protected static void init() throws IOException, URISyntaxException {
|
||||
TemplateListener.reset();
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder() {
|
||||
return builder(null);
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, null);
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String namespace) {
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId, String namespace) {
|
||||
return Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace == null ? "kestra.test" : namespace)
|
||||
.tenantId(tenantId)
|
||||
.tasks(Collections.singletonList(Return.builder().id("test").type(Return.class.getName()).format(Property.ofValue("test")).build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
|
||||
Optional<Template> full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
Optional<Template> full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
|
||||
full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
Template template1 = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
Template template2 = Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.tenantId(tenant)
|
||||
.namespace("kestra.test.template").build();
|
||||
|
||||
templateRepository.create(template1);
|
||||
templateRepository.create(template2);
|
||||
|
||||
List<Template> templates = templateRepository.findByNamespace(null, template1.getNamespace());
|
||||
List<Template> templates = templateRepository.findByNamespace(tenant, template1.getNamespace());
|
||||
assertThat(templates.size()).isGreaterThanOrEqualTo(1);
|
||||
templates = templateRepository.findByNamespace(null, template2.getNamespace());
|
||||
templates = templateRepository.findByNamespace(tenant, template2.getNamespace());
|
||||
assertThat(templates.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
Template save = templateRepository.create(template);
|
||||
|
||||
assertThat(save.getId()).isEqualTo(template.getId());
|
||||
@@ -84,41 +96,42 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
long saveCount = templateRepository.findAll(null).size();
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAll(tenant).size();
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAll(null).size();
|
||||
long size = templateRepository.findAll(tenant).size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAll(null).size()).isEqualTo(saveCount);
|
||||
assertThat((long) templateRepository.findAll(tenant).size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAllForAllTenants().size();
|
||||
Template template = builder().build();
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAllForAllTenants().size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAllForAllTenants().size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find() {
|
||||
Template template1 = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
templateRepository.create(template1);
|
||||
Template template2 = builder().build();
|
||||
Template template2 = builder(tenant).build();
|
||||
templateRepository.create(template2);
|
||||
Template template3 = builder().build();
|
||||
Template template3 = builder(tenant).build();
|
||||
templateRepository.create(template3);
|
||||
|
||||
// with pageable
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, null, "kestra.test");
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, tenant, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
// without pageable
|
||||
save = templateRepository.find(null, null, "kestra.test");
|
||||
save = templateRepository.find(null, tenant, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
templateRepository.delete(template1);
|
||||
@@ -126,31 +139,45 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
templateRepository.delete(template3);
|
||||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractTemplateRepositoryTest.class);
|
||||
|
||||
@Test
|
||||
void delete() {
|
||||
Template template = builder().build();
|
||||
protected void delete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
|
||||
Template save = templateRepository.create(template);
|
||||
templateRepository.delete(save);
|
||||
|
||||
assertThat(templateRepository.findById(null, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
assertThat(templateRepository.findById(tenant, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
|
||||
assertThat(TemplateListener.getEmits().size()).isEqualTo(2);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> {
|
||||
LOG.info("-------------> number of event: {}", TemplateListener.getEmits(tenant).size());
|
||||
return TemplateListener.getEmits(tenant).size() == 2;
|
||||
|
||||
}, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class TemplateListener implements ApplicationEventListener<CrudEvent<Template>> {
|
||||
private static List<CrudEvent<Template>> emits = new ArrayList<>();
|
||||
private static List<CrudEvent<Template>> emits = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<Template> event) {
|
||||
emits.add(event);
|
||||
//The instanceOf is required because Micronaut may send non Template event via this method
|
||||
if ((event.getModel() != null && event.getModel() instanceof Template) ||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof Template)) {
|
||||
emits.add(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static List<CrudEvent<Template>> getEmits() {
|
||||
return emits;
|
||||
public static List<CrudEvent<Template>> getEmits(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getModel() != null && e.getModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
|
||||
@@ -9,6 +9,7 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -24,7 +25,6 @@ import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@@ -35,8 +35,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Inject
|
||||
protected TriggerRepositoryInterface triggerRepository;
|
||||
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger() {
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger(String tenantId) {
|
||||
return Trigger.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId(IdUtils.create())
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.triggerId(IdUtils.create())
|
||||
@@ -44,9 +45,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
.date(ZonedDateTime.now());
|
||||
}
|
||||
|
||||
protected static Trigger generateDefaultTrigger(){
|
||||
protected static Trigger generateDefaultTrigger(String tenantId){
|
||||
Trigger trigger = Trigger.builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.triggerId("triggerId")
|
||||
.namespace("trigger.namespace")
|
||||
.flowId("flowId")
|
||||
@@ -59,9 +60,10 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -69,9 +71,10 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_async(QueryFilter filter){
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
|
||||
List<Trigger> entries = triggerRepository.find(MAIN_TENANT, List.of(filter)).collectList().block();
|
||||
List<Trigger> entries = triggerRepository.find(tenant, List.of(filter)).collectList().block();
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -92,7 +95,7 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -110,7 +113,8 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger(tenant);
|
||||
|
||||
Optional<Trigger> findLast = triggerRepository.findLast(builder.build());
|
||||
assertThat(findLast.isPresent()).isFalse();
|
||||
@@ -130,47 +134,47 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
assertThat(findLast.get().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
|
||||
triggerRepository.save(trigger().build());
|
||||
triggerRepository.save(trigger().build());
|
||||
Trigger searchedTrigger = trigger().build();
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
Trigger searchedTrigger = trigger(tenant).build();
|
||||
triggerRepository.save(searchedTrigger);
|
||||
|
||||
List<Trigger> all = triggerRepository.findAllForAllTenants();
|
||||
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
assertThat(all.size()).isGreaterThanOrEqualTo(4);
|
||||
|
||||
all = triggerRepository.findAll(null);
|
||||
all = triggerRepository.findAll(tenant);
|
||||
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
|
||||
String namespacePrefix = "io.kestra.another";
|
||||
String namespace = namespacePrefix + ".ns";
|
||||
Trigger trigger = trigger().namespace(namespace).build();
|
||||
Trigger trigger = trigger(tenant).namespace(namespace).build();
|
||||
triggerRepository.save(trigger);
|
||||
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, null, null);
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(4);
|
||||
assertThat(find.getFirst().getNamespace()).isEqualTo(namespace);
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, searchedTrigger.getFlowId(), null);
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, searchedTrigger.getFlowId(), null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getFlowId()).isEqualTo(searchedTrigger.getFlowId());
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, null, namespacePrefix, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, tenant, namespacePrefix, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
|
||||
// Full text search is on namespace, flowId, triggerId, executionId
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
}
|
||||
@@ -178,15 +182,17 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(Trigger
|
||||
.builder()
|
||||
.tenantId(tenant)
|
||||
.triggerId(IdUtils.create())
|
||||
.flowId(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.build()
|
||||
);
|
||||
// When
|
||||
int count = triggerRepository.count(null);
|
||||
int count = triggerRepository.count(tenant);
|
||||
// Then
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@@ -1,88 +1,92 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
|
||||
class ExecutionFixture {
|
||||
public static final Execution EXECUTION_1 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
public static Execution EXECUTION_1(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static final Execution EXECUTION_2 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
public static Execution EXECUTION_2(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static final Execution EXECUTION_TEST = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
public static Execution EXECUTION_TEST(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
@@ -13,36 +9,36 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.plugin.core.flow.EachSequentialTest;
|
||||
import io.kestra.plugin.core.flow.FlowCaseTest;
|
||||
import io.kestra.plugin.core.flow.ForEachItemCaseTest;
|
||||
import io.kestra.plugin.core.flow.PauseTest;
|
||||
import io.kestra.plugin.core.flow.LoopUntilCaseTest;
|
||||
import io.kestra.plugin.core.flow.WorkingDirectoryTest;
|
||||
import io.kestra.plugin.core.flow.*;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
//@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
|
||||
// must be per-class to allow calling once init() which took a lot of time
|
||||
public abstract class AbstractRunnerTest {
|
||||
|
||||
public static final String TENANT_1 = "tenant1";
|
||||
public static final String TENANT_2 = "tenant2";
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
|
||||
protected QueueInterface<LogEntry> logsQueue;
|
||||
|
||||
@Inject
|
||||
private RestartCaseTest restartCaseTest;
|
||||
protected RestartCaseTest restartCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowTriggerCaseTest flowTriggerCaseTest;
|
||||
@@ -54,13 +50,13 @@ public abstract class AbstractRunnerTest {
|
||||
private PluginDefaultsCaseTest pluginDefaultsCaseTest;
|
||||
|
||||
@Inject
|
||||
private FlowCaseTest flowCaseTest;
|
||||
protected FlowCaseTest flowCaseTest;
|
||||
|
||||
@Inject
|
||||
private WorkingDirectoryTest.Suite workingDirectoryTest;
|
||||
|
||||
@Inject
|
||||
private PauseTest.Suite pauseTest;
|
||||
protected PauseTest.Suite pauseTest;
|
||||
|
||||
@Inject
|
||||
private SkipExecutionCaseTest skipExecutionCaseTest;
|
||||
@@ -72,10 +68,10 @@ public abstract class AbstractRunnerTest {
|
||||
protected LoopUntilCaseTest loopUntilTestCaseTest;
|
||||
|
||||
@Inject
|
||||
private FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
protected FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
|
||||
@Inject
|
||||
private ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
protected ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowInputOutput flowIO;
|
||||
@@ -84,7 +80,7 @@ public abstract class AbstractRunnerTest {
|
||||
private SLATestCase slaTestCase;
|
||||
|
||||
@Inject
|
||||
private ChangeStateTestCase changeStateTestCase;
|
||||
protected ChangeStateTestCase changeStateTestCase;
|
||||
|
||||
@Inject
|
||||
private AfterExecutionTestCase afterExecutionTestCase;
|
||||
@@ -115,7 +111,7 @@ public abstract class AbstractRunnerTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(8);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/parallel-nested.yaml")
|
||||
void parallelNested(Execution execution) {
|
||||
assertThat(execution.getTaskRunList()).hasSize(11);
|
||||
@@ -157,27 +153,27 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedThenSuccess();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
void replay() throws Exception {
|
||||
restartCaseTest.replay();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/failed-first.yaml"})
|
||||
void restartMultiple() throws Exception {
|
||||
restartCaseTest.restartMultiple();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_always_failed.yaml"})
|
||||
void restartFailedThenFailureWithGlobalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithGlobalErrors();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_local_errors.yaml"})
|
||||
void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
protected void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithLocalErrors();
|
||||
}
|
||||
|
||||
@@ -199,29 +195,27 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedWithAfterExecution();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
"flows/valids/trigger-flow-listener.yaml",
|
||||
"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-flow.yaml"})
|
||||
"flows/valids/trigger-flow.yaml"}, tenantId = "listener-tenant")
|
||||
void flowTrigger() throws Exception {
|
||||
flowTriggerCaseTest.trigger();
|
||||
flowTriggerCaseTest.trigger("listener-tenant");
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on CI but never fail locally
|
||||
@Test // flaky on CI but never fail locally
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-pause.yaml",
|
||||
"flows/valids/trigger-flow-with-pause.yaml"})
|
||||
void flowTriggerWithPause() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithPause();
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"}, tenantId = "trigger-tenant")
|
||||
void flowTriggerWithConcurrencyLimit() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit();
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit("trigger-tenant");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -232,12 +226,12 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.trigger();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
@Test // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-c.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"})
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"}, tenantId = "condition-tenant")
|
||||
void multipleConditionTriggerFailed() throws Exception {
|
||||
multipleConditionTriggerCaseTest.failed();
|
||||
multipleConditionTriggerCaseTest.failed("condition-tenant");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -248,12 +242,13 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditions();
|
||||
}
|
||||
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
@LoadFlows(value = {"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-a.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"})
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"}, tenantId = TENANT_1)
|
||||
void flowTriggerPreconditionsMergeOutputs() throws Exception {
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs();
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -262,7 +257,7 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.flowTriggerOnPaused();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/each-null.yaml"})
|
||||
void eachWithNull() throws Exception {
|
||||
EachSequentialTest.eachNullTest(runnerUtils, logsQueue);
|
||||
@@ -274,7 +269,7 @@ public abstract class AbstractRunnerTest {
|
||||
pluginDefaultsCaseTest.taskDefaults();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
@@ -283,19 +278,19 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_1)
|
||||
void flowWaitFailed() throws Exception {
|
||||
flowCaseTest.waitFailed();
|
||||
flowCaseTest.waitFailed(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_2)
|
||||
public void invalidOutputs() throws Exception {
|
||||
flowCaseTest.invalidOutputs();
|
||||
flowCaseTest.invalidOutputs(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -305,9 +300,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/working-directory.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/working-directory.yaml"}, tenantId = TENANT_1)
|
||||
public void workerFailed() throws Exception {
|
||||
workingDirectoryTest.failed(runnerUtils);
|
||||
workingDirectoryTest.failed(TENANT_1, runnerUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -322,7 +317,7 @@ public abstract class AbstractRunnerTest {
|
||||
workingDirectoryTest.cache(runnerUtils);
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on MySQL
|
||||
@Test // flaky on MySQL
|
||||
@LoadFlows({"flows/valids/pause-test.yaml"})
|
||||
public void pauseRun() throws Exception {
|
||||
pauseTest.run(runnerUtils);
|
||||
@@ -358,44 +353,44 @@ public abstract class AbstractRunnerTest {
|
||||
skipExecutionCaseTest.skipExecution();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item.yaml"})
|
||||
protected void forEachItem() throws Exception {
|
||||
forEachItemCaseTest.forEachItem();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item.yaml"})
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item.yaml"}, tenantId = TENANT_1)
|
||||
protected void forEachItemEmptyItems() throws Exception {
|
||||
forEachItemCaseTest.forEachItemEmptyItems();
|
||||
forEachItemCaseTest.forEachItemEmptyItems(TENANT_1);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow-failed.yaml",
|
||||
"flows/valids/for-each-item-failed.yaml"})
|
||||
protected void forEachItemFailed() throws Exception {
|
||||
forEachItemCaseTest.forEachItemFailed();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-outputs-subflow.yaml",
|
||||
"flows/valids/for-each-item-outputs.yaml"})
|
||||
protected void forEachItemSubflowOutputs() throws Exception {
|
||||
forEachItemCaseTest.forEachItemWithSubflowOutputs();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows({"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"})
|
||||
@Test // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows(value = {"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"}, tenantId = TENANT_1)
|
||||
void restartForEachItem() throws Exception {
|
||||
forEachItemCaseTest.restartForEachItem();
|
||||
forEachItemCaseTest.restartForEachItem(TENANT_1);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"})
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"}, tenantId = TENANT_1)
|
||||
protected void forEachItemInIf() throws Exception {
|
||||
forEachItemCaseTest.forEachItemInIf();
|
||||
forEachItemCaseTest.forEachItemInIf(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -436,9 +431,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"})
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"}, tenantId = TENANT_1)
|
||||
protected void flowConcurrencyWithForEachItem() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem();
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -453,6 +448,12 @@ public abstract class AbstractRunnerTest {
|
||||
flowConcurrencyCaseTest.flowConcurrencyQueueAfterExecution();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"}, tenantId = TENANT_1)
|
||||
void flowConcurrencySubflow() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencySubflow(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/executable-fail.yml")
|
||||
void badExecutable(Execution execution) {
|
||||
@@ -505,9 +506,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/minimal.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/minimal.yaml"}, tenantId = TENANT_1)
|
||||
void shouldScheduleOnDate() throws Exception {
|
||||
scheduleDateCaseTest.shouldScheduleOnDate();
|
||||
scheduleDateCaseTest.shouldScheduleOnDate(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -529,15 +530,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_1)
|
||||
void executionConditionSLAShouldCancel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldCancel();
|
||||
slaTestCase.executionConditionSLAShouldCancel(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_2)
|
||||
void executionConditionSLAShouldLabel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldLabel();
|
||||
slaTestCase.executionConditionSLAShouldLabel(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -557,15 +558,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/failed-first.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/failed-first.yaml", tenantId = TENANT_1)
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
changeStateTestCase.changeStateShouldEndsInSuccess(execution);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"}, tenantId = TENANT_2)
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess();
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -3,25 +3,18 @@ package io.kestra.core.runners;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Singleton
|
||||
public class ChangeStateTestCase {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests";
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@@ -29,11 +22,7 @@ public class ChangeStateTestCase {
|
||||
private ExecutionService executionService;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
private QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
@@ -41,73 +30,40 @@ public class ChangeStateTestCase {
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the last execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
Execution markedAs = executionService.markAs(execution, flow, execution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
executionQueue.emit(markedAs);
|
||||
Execution lastExecution = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), markedAs);
|
||||
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
// await for the subflow execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if ("failed-first".equals(exec.getFlowId()) && exec.getState().getCurrent() == State.Type.FAILED) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess(String tenantId) throws Exception {
|
||||
// run the parent flow
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-parent-of-failed");
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// assert on the subflow
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the parent execution
|
||||
CountDownLatch parentLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastParentExecution = new AtomicReference<>();
|
||||
receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().isTerminated()) {
|
||||
lastParentExecution.set(exec);
|
||||
parentLatch.countDown();
|
||||
}
|
||||
});
|
||||
Execution lastExecution = runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.FAILED), tenantId, NAMESPACE, "failed-first");
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// restart the subflow
|
||||
Flow flow = flowRepository.findByExecution(lastExecution.get());
|
||||
Execution markedAs = executionService.markAs(lastExecution.get(), flow, lastExecution.get().getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
executionQueue.emit(markedAs);
|
||||
Flow flow = flowRepository.findByExecution(lastExecution);
|
||||
Execution markedAs = executionService.markAs(lastExecution, flow, lastExecution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().isTerminated(), markedAs);
|
||||
|
||||
//We wait for the subflow execution to pass from failed to success
|
||||
Execution lastParentExecution = runnerUtils.awaitFlowExecution(e ->
|
||||
e.getTaskRunList().getFirst().getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
|
||||
// assert for the parent flow
|
||||
assertThat(parentLatch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastParentExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastParentExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
public class EmptyVariablesTest {
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.plugin.core.flow.Pause;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
import org.slf4j.event.Level;
|
||||
@@ -40,6 +41,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
@Slf4j
|
||||
@KestraTest(startRunner = true)
|
||||
class ExecutionServiceTest {
|
||||
|
||||
public static final String TENANT_1 = "tenant1";
|
||||
public static final String TENANT_2 = "tenant2";
|
||||
public static final String TENANT_3 = "tenant3";
|
||||
@Inject
|
||||
ExecutionService executionService;
|
||||
|
||||
@@ -53,7 +58,7 @@ class ExecutionServiceTest {
|
||||
LogRepositoryInterface logRepository;
|
||||
|
||||
@Inject
|
||||
RunnerUtils runnerUtils;
|
||||
TestRunnerUtils runnerUtils;
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
|
||||
@@ -75,13 +80,13 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/restart_last_failed.yaml"}, tenantId = TENANT_1)
|
||||
void restartSimpleRevision() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart_last_failed");
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart_last_failed");
|
||||
assertThat(execution.getTaskRunList()).hasSize(3);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
FlowWithSource flow = flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests", "restart_last_failed").orElseThrow();
|
||||
FlowWithSource flow = flowRepository.findByIdWithSource(TENANT_1, "io.kestra.tests", "restart_last_failed").orElseThrow();
|
||||
flowRepository.update(
|
||||
GenericFlow.of(flow),
|
||||
flow.updateTask(
|
||||
@@ -124,9 +129,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_1)
|
||||
void restartFlowable2() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
Execution restart = executionService.restart(execution, null);
|
||||
@@ -177,9 +182,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/logs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/logs.yaml"}, tenantId = TENANT_1)
|
||||
void replaySimple() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "logs");
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "logs");
|
||||
assertThat(execution.getTaskRunList()).hasSize(5);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
@@ -196,9 +201,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_2)
|
||||
void replayFlowable() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
|
||||
Execution execution = runnerUtils.runOne(TENANT_2, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
|
||||
assertThat(execution.getTaskRunList()).hasSize(20);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
@@ -213,6 +218,7 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.getLabels()).contains(new Label(Label.REPLAY, "true"));
|
||||
}
|
||||
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/parallel-nested.yaml"})
|
||||
void replayParallel() throws Exception {
|
||||
@@ -234,7 +240,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_2)
|
||||
void replayEachSeq(Execution execution) throws Exception {
|
||||
assertThat(execution.getTaskRunList()).hasSize(23);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
@@ -253,7 +259,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_1)
|
||||
void replayEachSeq2(Execution execution) throws Exception {
|
||||
assertThat(execution.getTaskRunList()).hasSize(23);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
@@ -312,9 +318,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/each-parallel-nested.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/each-parallel-nested.yaml"}, tenantId = TENANT_1)
|
||||
void markAsEachPara() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "each-parallel-nested");
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "each-parallel-nested");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(11);
|
||||
@@ -364,9 +370,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/pause-test.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/pause-test.yaml"}, tenantId = TENANT_1)
|
||||
void resumePausedToKilling() throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause-test");
|
||||
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause-test");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
@@ -379,7 +385,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/logs.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_2)
|
||||
void deleteExecution(Execution execution) throws IOException, TimeoutException {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
@@ -391,7 +397,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/logs.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_3)
|
||||
void deleteExecutionKeepLogs(Execution execution) throws IOException, TimeoutException {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
@@ -431,9 +437,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/pause_no_tasks.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/pause_no_tasks.yaml"}, tenantId = TENANT_1)
|
||||
void killToState() throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause_no_tasks");
|
||||
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause_no_tasks");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
Execution killed = executionService.kill(execution, flow, Optional.of(State.Type.CANCELLED));
|
||||
|
||||
@@ -18,11 +18,14 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(rebuildContext = true)
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class FilesServiceTest {
|
||||
@Inject
|
||||
private TestRunContextFactory runContextFactory;
|
||||
|
||||
@@ -3,19 +3,15 @@ package io.kestra.core.runners;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.History;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
@@ -25,24 +21,21 @@ import java.net.URISyntaxException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class FlowConcurrencyCaseTest {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests";
|
||||
@Inject
|
||||
private StorageInterface storageInterface;
|
||||
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
@@ -50,354 +43,168 @@ public class FlowConcurrencyCaseTest {
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
private ExecutionService executionService;
|
||||
|
||||
public void flowConcurrencyCancel() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel");
|
||||
public void flowConcurrencyCancel() throws TimeoutException, QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel");
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
}
|
||||
|
||||
public void flowConcurrencyFail() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail");
|
||||
public void flowConcurrencyFail() throws TimeoutException, QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail");
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueue() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
|
||||
public void flowConcurrencyQueue() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", Optional.empty())
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
executionQueue.emit(execution2);
|
||||
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueuePause() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<String> firstExecutionId = new AtomicReference<>();
|
||||
var firstExecutionResult = new AtomicReference<Execution>();
|
||||
var secondExecutionResult = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch firstExecutionLatch = new CountDownLatch(1);
|
||||
CountDownLatch secondExecutionLatch = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (!"flow-concurrency-queue-pause".equals(e.getLeft().getFlowId())){
|
||||
return;
|
||||
}
|
||||
String currentId = e.getLeft().getId();
|
||||
Type currentState = e.getLeft().getState().getCurrent();
|
||||
if (firstExecutionId.get() == null) {
|
||||
firstExecutionId.set(currentId);
|
||||
}
|
||||
|
||||
if (currentId.equals(firstExecutionId.get())) {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
firstExecutionResult.set(e.getLeft());
|
||||
firstExecutionLatch.countDown();
|
||||
}
|
||||
} else {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
secondExecutionResult.set(e.getLeft());
|
||||
secondExecutionLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause");
|
||||
public void flowConcurrencyQueuePause() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause");
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause", Optional.empty())
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
executionQueue.emit(execution2);
|
||||
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
|
||||
assertThat(execution1.getState().isPaused()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
assertTrue(firstExecutionLatch.await(10, TimeUnit.SECONDS));
|
||||
assertTrue(secondExecutionLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyCancelPause() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<String> firstExecutionId = new AtomicReference<>();
|
||||
var firstExecutionResult = new AtomicReference<Execution>();
|
||||
var secondExecutionResult = new AtomicReference<Execution>();
|
||||
CountDownLatch firstExecLatch = new CountDownLatch(1);
|
||||
CountDownLatch secondExecLatch = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (!"flow-concurrency-cancel-pause".equals(e.getLeft().getFlowId())){
|
||||
return;
|
||||
}
|
||||
String currentId = e.getLeft().getId();
|
||||
Type currentState = e.getLeft().getState().getCurrent();
|
||||
if (firstExecutionId.get() == null) {
|
||||
firstExecutionId.set(currentId);
|
||||
}
|
||||
if (currentId.equals(firstExecutionId.get())) {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
firstExecutionResult.set(e.getLeft());
|
||||
firstExecLatch.countDown();
|
||||
}
|
||||
} else {
|
||||
if (currentState == State.Type.CANCELLED) {
|
||||
secondExecutionResult.set(e.getLeft());
|
||||
secondExecLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause");
|
||||
public void flowConcurrencyCancelPause() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause");
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause", Optional.empty())
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
executionQueue.emit(execution2);
|
||||
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.CANCELLED), execution2);
|
||||
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
|
||||
assertThat(execution1.getState().isPaused()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
assertTrue(firstExecLatch.await(10, TimeUnit.SECONDS));
|
||||
assertTrue(secondExecLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
|
||||
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
|
||||
}
|
||||
|
||||
public void flowConcurrencyWithForEachItem() throws TimeoutException, QueueException, InterruptedException, URISyntaxException, IOException {
|
||||
URI file = storageUpload();
|
||||
public void flowConcurrencyWithForEachItem(String tenantId) throws QueueException, URISyntaxException, IOException {
|
||||
URI file = storageUpload(tenantId);
|
||||
Map<String, Object> inputs = Map.of("file", file.toString(), "batch", 4);
|
||||
Execution forEachItem = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-for-each-item", null,
|
||||
Execution forEachItem = runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-for-each-item", null,
|
||||
(flow, execution1) -> flowIO.readExecutionInputs(flow, execution1, inputs), Duration.ofSeconds(5));
|
||||
assertThat(forEachItem.getState().getCurrent()).isEqualTo(Type.RUNNING);
|
||||
|
||||
Set<String> executionIds = new HashSet<>();
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if ("flow-concurrency-queue".equals(e.getLeft().getFlowId()) && e.getLeft().getState().isRunning()) {
|
||||
executionIds.add(e.getLeft().getId());
|
||||
}
|
||||
});
|
||||
|
||||
// wait a little to be sure there are not too many executions started
|
||||
Thread.sleep(500);
|
||||
|
||||
assertThat(executionIds).hasSize(1);
|
||||
receive.blockLast();
|
||||
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getId().equals(forEachItem.getId()) && e.getState().isTerminated(), () -> {}, Duration.ofSeconds(10));
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getState().isTerminated(),forEachItem);
|
||||
assertThat(terminated.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
|
||||
List<Execution> executions = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-queue");
|
||||
|
||||
assertThat(executions).extracting(e -> e.getState().getCurrent()).containsOnly(Type.SUCCESS);
|
||||
assertThat(executions.stream()
|
||||
.map(e -> e.getState().getHistories())
|
||||
.flatMap(List::stream)
|
||||
.map(History::getState)
|
||||
.toList()).contains(Type.QUEUED);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueueRestarted() throws Exception {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE,
|
||||
"flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", Optional.empty())
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-fail", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
executionQueue.emit(execution2);
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.RUNNING), execution2);
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(2);
|
||||
AtomicReference<Execution> failedExecution = new AtomicReference<>();
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
|
||||
failedExecution.set(e.getLeft());
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertThat(failedExecution.get()).isNotNull();
|
||||
// here the first fail and the second is now running.
|
||||
// we restart the first one, it should be queued then fail again.
|
||||
Execution restarted = executionService.restart(failedExecution.get(), null);
|
||||
executionQueue.emit(restarted);
|
||||
Execution failedExecution = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution1);
|
||||
Execution restarted = executionService.restart(failedExecution, null);
|
||||
Execution executionResult1 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), restarted);
|
||||
Execution executionResult2 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution2);
|
||||
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
// it should have been queued after restarted
|
||||
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
|
||||
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
|
||||
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueueAfterExecution() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
|
||||
public void flowConcurrencyQueueAfterExecution() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", Optional.empty())
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
executionQueue.emit(execution2);
|
||||
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
private URI storageUpload() throws URISyntaxException, IOException {
|
||||
public void flowConcurrencySubflow(String tenantId) throws TimeoutException, QueueException {
|
||||
runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-subflow", null, null, Duration.ofSeconds(30));
|
||||
runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
|
||||
|
||||
List<Execution> subFlowExecs = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-cancel");
|
||||
assertThat(subFlowExecs).extracting(e -> e.getState().getCurrent()).containsExactlyInAnyOrder(Type.SUCCESS, Type.CANCELLED);
|
||||
|
||||
// run another execution to be sure that everything work (purge is correctly done)
|
||||
Execution execution3 = runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
|
||||
assertThat(execution3.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "flow-concurrency-cancel");
|
||||
}
|
||||
|
||||
private URI storageUpload(String tenantId) throws URISyntaxException, IOException {
|
||||
File tempFile = File.createTempFile("file", ".txt");
|
||||
|
||||
Files.write(tempFile.toPath(), content());
|
||||
|
||||
return storageInterface.put(
|
||||
MAIN_TENANT,
|
||||
tenantId,
|
||||
null,
|
||||
new URI("/file/storage/file.txt"),
|
||||
new FileInputStream(tempFile)
|
||||
|
||||
@@ -4,19 +4,22 @@ import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import lombok.SneakyThrows;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.FlowListenersInterface;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import jakarta.inject.Inject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@@ -24,11 +27,11 @@ abstract public class FlowListenersTest {
|
||||
@Inject
|
||||
protected FlowRepositoryInterface flowRepository;
|
||||
|
||||
protected static FlowWithSource create(String flowId, String taskId) {
|
||||
protected static FlowWithSource create(String tenantId, String flowId, String taskId) {
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id(flowId)
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.revision(1)
|
||||
.tasks(Collections.singletonList(Return.builder()
|
||||
.id(taskId)
|
||||
@@ -39,88 +42,65 @@ abstract public class FlowListenersTest {
|
||||
return flow.toBuilder().source(flow.sourceOrGenerateIfNull()).build();
|
||||
}
|
||||
|
||||
public void suite(FlowListenersInterface flowListenersService) {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(FlowListenersTest.class);
|
||||
|
||||
public void suite(FlowListenersInterface flowListenersService) throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowListenersService.run();
|
||||
|
||||
AtomicInteger count = new AtomicInteger();
|
||||
var ref = new Ref();
|
||||
|
||||
flowListenersService.listen(flows -> {
|
||||
count.set(flows.size());
|
||||
ref.countDownLatch.countDown();
|
||||
});
|
||||
flowListenersService.listen(flows -> count.set(getFlowsForTenant(flowListenersService, tenant).size()));
|
||||
|
||||
// initial state
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isZero();
|
||||
assertThat(flowListenersService.flows().size()).isZero();
|
||||
});
|
||||
LOG.info("-----------> wait for zero");
|
||||
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
|
||||
|
||||
// resend on startup done for kafka
|
||||
LOG.info("-----------> wait for zero kafka");
|
||||
if (flowListenersService.getClass().getName().equals("io.kestra.ee.runner.kafka.KafkaFlowListeners")) {
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isZero();
|
||||
assertThat(flowListenersService.flows().size()).isZero();
|
||||
});
|
||||
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
|
||||
}
|
||||
|
||||
// create first
|
||||
FlowWithSource first = create("first_" + IdUtils.create(), "test");
|
||||
FlowWithSource firstUpdated = create(first.getId(), "test2");
|
||||
LOG.info("-----------> create fist flow");
|
||||
FlowWithSource first = create(tenant, "first_" + IdUtils.create(), "test");
|
||||
FlowWithSource firstUpdated = create(tenant, first.getId(), "test2");
|
||||
|
||||
|
||||
flowRepository.create(GenericFlow.of(first));
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
});
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
|
||||
// create the same id than first, no additional flows
|
||||
first = flowRepository.update(GenericFlow.of(firstUpdated), first);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
//assertThat(flowListenersService.flows().getFirst().getFirst().getId(), is("test2"));
|
||||
});
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
|
||||
FlowWithSource second = create("second_" + IdUtils.create(), "test");
|
||||
FlowWithSource second = create(tenant, "second_" + IdUtils.create(), "test");
|
||||
// create a new one
|
||||
flowRepository.create(GenericFlow.of(second));
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(2);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(2);
|
||||
});
|
||||
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
|
||||
|
||||
// delete first
|
||||
FlowWithSource deleted = flowRepository.delete(first);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
});
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
|
||||
// restore must works
|
||||
flowRepository.create(GenericFlow.of(first));
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(2);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(2);
|
||||
});
|
||||
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
|
||||
|
||||
FlowWithSource withTenant = first.toBuilder().tenantId("some-tenant").build();
|
||||
flowRepository.create(GenericFlow.of(withTenant));
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(3);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(3);
|
||||
});
|
||||
}
|
||||
|
||||
public static class Ref {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
public List<FlowWithSource> getFlowsForTenant(FlowListenersInterface flowListenersService, String tenantId){
|
||||
return flowListenersService.flows().stream()
|
||||
.filter(f -> tenantId.equals(f.getTenantId()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private void wait(Ref ref, Runnable run) {
|
||||
ref.countDownLatch.await(60, TimeUnit.SECONDS);
|
||||
run.run();
|
||||
ref.countDownLatch = new CountDownLatch(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,82 +2,61 @@ package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class FlowTriggerCaseTest {
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests.trigger";
|
||||
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
public void trigger() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(3);
|
||||
AtomicReference<Execution> flowListener = new AtomicReference<>();
|
||||
AtomicReference<Execution> flowListenerNoInput = new AtomicReference<>();
|
||||
AtomicReference<Execution> flowListenerNamespace = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
if (flowListenerNoInput.get() == null && execution.getFlowId().equals("trigger-flow-listener-no-inputs")) {
|
||||
flowListenerNoInput.set(execution);
|
||||
countDownLatch.countDown();
|
||||
} else if (flowListener.get() == null && execution.getFlowId().equals("trigger-flow-listener")) {
|
||||
flowListener.set(execution);
|
||||
countDownLatch.countDown();
|
||||
} else if (flowListenerNamespace.get() == null && execution.getFlowId().equals("trigger-flow-listener-namespace-condition")) {
|
||||
flowListenerNamespace.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger", "trigger-flow");
|
||||
public void trigger(String tenantId) throws InterruptedException, TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "trigger-flow");
|
||||
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
Execution flowListenerNoInput = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener-no-inputs");
|
||||
Execution flowListener = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener");
|
||||
Execution flowListenerNamespace = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener-namespace-condition");
|
||||
|
||||
assertThat(flowListener.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(flowListener.get().getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
|
||||
assertThat(flowListenerNoInput.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
assertThat(flowListenerNoInput.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(flowListener.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListener.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(flowListener.getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
|
||||
assertThat(flowListener.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListener.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
assertThat(flowListener.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
|
||||
assertThat(flowListenerNamespace.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(flowListenerNoInput.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
assertThat(flowListenerNoInput.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(flowListenerNamespace.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNamespace.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
// it will be triggered for 'trigger-flow' or any of the 'trigger-flow-listener*', so we only assert that it's one of them
|
||||
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("flowId"))
|
||||
assertThat(flowListenerNamespace.getTrigger().getVariables().get("flowId"))
|
||||
.satisfiesAnyOf(
|
||||
arg -> assertThat(arg).isEqualTo("trigger-flow"),
|
||||
arg -> assertThat(arg).isEqualTo("trigger-flow-listener-no-inputs"),
|
||||
@@ -85,56 +64,43 @@ public class FlowTriggerCaseTest {
|
||||
);
|
||||
}
|
||||
|
||||
public void triggerWithPause() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(4);
|
||||
List<Execution> flowListeners = new ArrayList<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-pause")) {
|
||||
flowListeners.add(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
public void triggerWithPause() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.pause", "trigger-flow-with-pause");
|
||||
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(3);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
|
||||
4,
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests.trigger.pause",
|
||||
"trigger-flow-listener-with-pause");
|
||||
|
||||
assertThat(flowListeners.size()).isEqualTo(4);
|
||||
assertThat(flowListeners.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(flowListeners.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
|
||||
assertThat(flowListeners.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(flowListeners.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
|
||||
assertThat(triggeredExec.size()).isEqualTo(4);
|
||||
List<Execution> sortedExecs = triggeredExec.stream()
|
||||
.sorted(Comparator.comparing(e -> e.getState().getEndDate().orElse(Instant.now())))
|
||||
.toList();
|
||||
assertThat(sortedExecs.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(sortedExecs.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
|
||||
assertThat(sortedExecs.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(sortedExecs.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
|
||||
}
|
||||
|
||||
public void triggerWithConcurrencyLimit() throws QueueException, TimeoutException, InterruptedException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(5);
|
||||
List<Execution> flowListeners = new ArrayList<>();
|
||||
public void triggerWithConcurrencyLimit(String tenantId) throws QueueException, TimeoutException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
Execution execution2 = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-concurrency-limit")) {
|
||||
flowListeners.add(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
|
||||
5,
|
||||
tenantId,
|
||||
"io.kestra.tests.trigger.concurrency",
|
||||
"trigger-flow-listener-with-concurrency-limit");
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
Execution execution2 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(flowListeners.size()).isEqualTo(5);
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(triggeredExec.size()).isEqualTo(5);
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -36,6 +39,7 @@ import java.util.concurrent.TimeoutException;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
public class InputsTest {
|
||||
@@ -44,7 +48,7 @@ public class InputsTest {
|
||||
private QueueInterface<LogEntry> logQueue;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public static Map<String, Object> inputs = ImmutableMap.<String, Object>builder()
|
||||
.put("string", "myString")
|
||||
@@ -90,8 +94,8 @@ public class InputsTest {
|
||||
@Inject
|
||||
private FlowInputOutput flowInputOutput;
|
||||
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map) {
|
||||
return typedInputs(map, flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "inputs").get());
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map, String tenantId) {
|
||||
return typedInputs(map, flowRepository.findById(tenantId, "io.kestra.tests", "inputs").get());
|
||||
}
|
||||
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map, Flow flow) {
|
||||
@@ -100,7 +104,7 @@ public class InputsTest {
|
||||
Execution.builder()
|
||||
.id("test")
|
||||
.namespace(flow.getNamespace())
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(flow.getTenantId())
|
||||
.flowRevision(1)
|
||||
.flowId(flow.getId())
|
||||
.build(),
|
||||
@@ -113,25 +117,25 @@ public class InputsTest {
|
||||
void missingRequired() {
|
||||
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
|
||||
inputs.put("string", null);
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs, MAIN_TENANT));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `string`, missing required input, but received `null`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant")
|
||||
void nonRequiredNoDefaultNoValueIsNull() {
|
||||
HashMap<String, Object> inputsWithMissingOptionalInput = new HashMap<>(inputs);
|
||||
inputsWithMissingOptionalInput.remove("bool");
|
||||
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput).containsKey("bool")).isTrue();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput).get("bool")).isNull();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").containsKey("bool")).isTrue();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").get("bool")).isNull();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant1")
|
||||
void allValidInputs() throws URISyntaxException, IOException {
|
||||
Map<String, Object> typeds = typedInputs(inputs);
|
||||
Map<String, Object> typeds = typedInputs(inputs, "tenant1");
|
||||
|
||||
assertThat(typeds.get("string")).isEqualTo("myString");
|
||||
assertThat(typeds.get("int")).isEqualTo(42);
|
||||
@@ -143,7 +147,7 @@ public class InputsTest {
|
||||
assertThat(typeds.get("time")).isEqualTo(LocalTime.parse("18:27:49"));
|
||||
assertThat(typeds.get("duration")).isEqualTo(Duration.parse("PT5M6S"));
|
||||
assertThat((URI) typeds.get("file")).isEqualTo(new URI("kestra:///io/kestra/tests/inputs/executions/test/inputs/file/application-test.yml"));
|
||||
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get(MAIN_TENANT, null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
|
||||
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get("tenant1", null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
|
||||
assertThat(typeds.get("json")).isEqualTo(Map.of("a", "b"));
|
||||
assertThat(typeds.get("uri")).isEqualTo("https://www.google.com");
|
||||
assertThat(((Map<String, Object>) typeds.get("nested")).get("string")).isEqualTo("a string");
|
||||
@@ -166,9 +170,9 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant2")
|
||||
void allValidTypedInputs() {
|
||||
Map<String, Object> typeds = typedInputs(inputs);
|
||||
Map<String, Object> typeds = typedInputs(inputs, "tenant2");
|
||||
typeds.put("int", 42);
|
||||
typeds.put("float", 42.42F);
|
||||
typeds.put("bool", false);
|
||||
@@ -181,10 +185,10 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant3")
|
||||
void inputFlow() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"tenant3",
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -201,165 +205,165 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant4")
|
||||
void inputValidatedStringBadValue() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("validatedString", "foo");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant4"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedString`, it must match the pattern");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant5")
|
||||
void inputValidatedIntegerBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedInt", "9");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant5"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be more than `10`, but received `9`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedInt", "21");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant5"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be less than `20`, but received `21`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant6")
|
||||
void inputValidatedDateBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDate", "2022-01-01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant6"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be after `2023-01-01`, but received `2022-01-01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDate", "2024-01-01");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant6"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be before `2023-12-31`, but received `2024-01-01`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant7")
|
||||
void inputValidatedDateTimeBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDateTime", "2022-01-01T00:00:00Z");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant7"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be after `2023-01-01T00:00:00Z`, but received `2022-01-01T00:00:00Z`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDateTime", "2024-01-01T00:00:00Z");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant7"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be before `2023-12-31T23:59:59Z`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant8")
|
||||
void inputValidatedDurationBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDuration", "PT1S");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant8"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be more than `PT10S`, but received `PT1S`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDuration", "PT30S");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant8"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be less than `PT20S`, but received `PT30S`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant9")
|
||||
void inputValidatedFloatBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedFloat", "0.01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant9"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be more than `0.1`, but received `0.01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedFloat", "1.01");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant9"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be less than `0.5`, but received `1.01`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant10")
|
||||
void inputValidatedTimeBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedTime", "00:00:01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant10"));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be after `01:00`, but received `00:00:01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedTime", "14:00:00");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant10"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be before `11:59:59`, but received `14:00:00`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant11")
|
||||
void inputFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("uri", "http:/bla");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant11"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `uri`, Expected `URI` but received `http:/bla`, but received `http:/bla`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant12")
|
||||
void inputEnumFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("enum", "INVALID");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant12"));
|
||||
|
||||
assertThat(e.getMessage()).isEqualTo("enum: Invalid input for `enum`, it must match the values `[ENUM_VALUE, OTHER_ONE]`, but received `INVALID`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant13")
|
||||
void inputArrayFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("array", "[\"s1\", \"s2\"]");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant13"));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `array`, Unable to parse array element as `INT` on `s1`, but received `[\"s1\", \"s2\"]`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant14")
|
||||
void inputEmptyJson() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("json", "{}");
|
||||
|
||||
Map<String, Object> typeds = typedInputs(map);
|
||||
Map<String, Object> typeds = typedInputs(map, "tenant14");
|
||||
|
||||
assertThat(typeds.get("json")).isInstanceOf(Map.class);
|
||||
assertThat(((Map<?, ?>) typeds.get("json")).size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant15")
|
||||
void inputEmptyJsonFlow() throws TimeoutException, QueueException {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("json", "{}");
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"tenant15",
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -375,12 +379,20 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/input-log-secret.yaml"})
|
||||
void shouldNotLogSecretInput() throws TimeoutException, QueueException {
|
||||
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {});
|
||||
@LoadFlows(value = {"flows/valids/input-log-secret.yaml"}, tenantId = "tenant16")
|
||||
void shouldNotLogSecretInput() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<LogEntry> logEntry = new AtomicReference<>();
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {
|
||||
LogEntry left = l.getLeft();
|
||||
if (left.getTenantId().equals("tenant16")){
|
||||
logEntry.set(left);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"tenant16",
|
||||
"io.kestra.tests",
|
||||
"input-log-secret",
|
||||
null,
|
||||
@@ -390,20 +402,21 @@ public class InputsTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
var logEntry = receive.blockLast();
|
||||
assertThat(logEntry).isNotNull();
|
||||
assertThat(logEntry.getMessage()).isEqualTo("These are my secrets: ****** - ******");
|
||||
receive.blockLast();
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
assertThat(logEntry.get()).isNotNull();
|
||||
assertThat(logEntry.get().getMessage()).isEqualTo("These are my secrets: ****** - ******");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant17")
|
||||
void fileInputWithFileDefault() throws IOException, QueueException, TimeoutException {
|
||||
HashMap<String, Object> newInputs = new HashMap<>(InputsTest.inputs);
|
||||
URI file = createFile();
|
||||
newInputs.put("file", file);
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"tenant17",
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -415,14 +428,14 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant18")
|
||||
void fileInputWithNsfile() throws IOException, QueueException, TimeoutException {
|
||||
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
|
||||
URI file = createNsFile(false);
|
||||
URI file = createNsFile(false, "tenant18");
|
||||
inputs.put("file", file);
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
MAIN_TENANT,
|
||||
"tenant18",
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -439,11 +452,11 @@ public class InputsTest {
|
||||
return tempFile.toPath().toUri();
|
||||
}
|
||||
|
||||
private URI createNsFile(boolean nsInAuthority) throws IOException {
|
||||
private URI createNsFile(boolean nsInAuthority, String tenantId) throws IOException {
|
||||
String namespace = "io.kestra.tests";
|
||||
String filePath = "file.txt";
|
||||
storageInterface.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storageInterface.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
storageInterface.createDirectory(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storageInterface.put(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
return URI.create("nsfile://" + (nsInAuthority ? namespace : "") + "/" + filePath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,15 +14,17 @@ import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
@KestraTest(startRunner = true)
|
||||
class ListenersTest {
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@@ -1,243 +1,168 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.core.repositories.ArrayListTotal;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class MultipleConditionTriggerCaseTest {
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
public static final String NAMESPACE = "io.kestra.tests.trigger";
|
||||
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
protected FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@Inject
|
||||
protected ApplicationContext applicationContext;
|
||||
|
||||
public void trigger() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(3);
|
||||
ConcurrentHashMap<String, Execution> ended = new ConcurrentHashMap<>();
|
||||
List<String> watchedExecutions = List.of("trigger-multiplecondition-flow-a",
|
||||
"trigger-multiplecondition-flow-b",
|
||||
"trigger-multiplecondition-listener"
|
||||
);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (watchedExecutions.contains(execution.getFlowId()) && execution.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
ended.put(execution.getId(), execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// first one
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-a", Duration.ofSeconds(60));
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-a");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// wait a little to be sure that the trigger is not launching execution
|
||||
Thread.sleep(1000);
|
||||
assertThat(ended.size()).isEqualTo(1);
|
||||
ArrayListTotal<Execution> flowBExecutions = executionRepository.findByFlowId(MAIN_TENANT,
|
||||
NAMESPACE, "trigger-multiplecondition-flow-b", Pageable.UNPAGED);
|
||||
ArrayListTotal<Execution> listenerExecutions = executionRepository.findByFlowId(MAIN_TENANT,
|
||||
NAMESPACE, "trigger-multiplecondition-listener", Pageable.UNPAGED);
|
||||
assertThat(flowBExecutions).isEmpty();
|
||||
assertThat(listenerExecutions).isEmpty();
|
||||
|
||||
// second one
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-b", Duration.ofSeconds(60));
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-b");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(ended.size()).isEqualTo(3);
|
||||
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-listener").orElseThrow();
|
||||
Execution triggerExecution = ended.entrySet()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().getFlowId().equals(flow.getId()))
|
||||
.findFirst()
|
||||
.map(Map.Entry::getValue)
|
||||
.orElseThrow();
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-listener");
|
||||
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo(
|
||||
NAMESPACE);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-multiplecondition-flow-b");
|
||||
}
|
||||
|
||||
public void failed() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> listener = new AtomicReference<>();
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getFlowId().equals("trigger-flow-listener-namespace-condition")
|
||||
&& execution.getState().getCurrent().isTerminated()) {
|
||||
listener.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
public void failed(String tenantId) throws InterruptedException, TimeoutException, QueueException {
|
||||
// first one
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-c", Duration.ofSeconds(60));
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-c");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait a little to be sure that the trigger is not launching execution
|
||||
Thread.sleep(1000);
|
||||
assertThat(listener.get()).isNull();
|
||||
ArrayListTotal<Execution> byFlowId = executionRepository.findByFlowId(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-d", Pageable.UNPAGED);
|
||||
assertThat(byFlowId).isEmpty();
|
||||
|
||||
// second one
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-d", Duration.ofSeconds(60));
|
||||
execution = runnerUtils.runOne(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-d");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
tenantId, NAMESPACE, "trigger-flow-listener-namespace-condition");
|
||||
|
||||
// trigger was not done
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(listener.get()).isNotNull();
|
||||
assertThat(listener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void flowTriggerPreconditions()
|
||||
throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-preconditions-flow-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
public void flowTriggerPreconditions() throws TimeoutException, QueueException {
|
||||
|
||||
// flowA
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// flowB: we trigger it two times, as flow-trigger-flow-preconditions-flow-listen is configured with resetOnSuccess: false it should be triggered two times
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
|
||||
"flow-trigger-preconditions-flow-b");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
|
||||
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
|
||||
}
|
||||
|
||||
public void flowTriggerPreconditionsMergeOutputs() throws QueueException, TimeoutException, InterruptedException {
|
||||
public void flowTriggerPreconditionsMergeOutputs(String tenantId) throws QueueException, TimeoutException {
|
||||
// we do the same as in flowTriggerPreconditions() but we trigger flows in the opposite order to be sure that outputs are merged
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-preconditions-flow-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// flowB
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
|
||||
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// flowA
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
tenantId, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
|
||||
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
|
||||
}
|
||||
|
||||
public void flowTriggerOnPaused()
|
||||
throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-paused-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
public void flowTriggerOnPaused() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.paused",
|
||||
"flow-trigger-paused-flow", Duration.ofSeconds(60));
|
||||
"flow-trigger-paused-flow");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, "io.kestra.tests.trigger.paused", "flow-trigger-paused-listen");
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@Singleton
|
||||
public class PluginDefaultsCaseTest {
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public void taskDefaults() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "plugin-defaults", Duration.ofSeconds(60));
|
||||
|
||||
@@ -4,29 +4,19 @@ import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
@@ -35,38 +25,30 @@ public class RestartCaseTest {
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private ExecutionService executionService;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
private QueueInterface<Execution> executionQueue;
|
||||
|
||||
public void restartFailedThenSuccess() throws Exception {
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "restart_last_failed").orElseThrow();
|
||||
|
||||
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId(), Duration.ofSeconds(60));
|
||||
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
|
||||
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(firstExecution.getTaskRunList()).hasSize(3);
|
||||
assertThat(firstExecution.getTaskRunList().get(2).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(3);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.SUCCESS && execution.getId().equals(firstExecution.getId()),
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(3);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
executionQueue.emit(restartedExec);
|
||||
}),
|
||||
Duration.ofSeconds(60)
|
||||
restartedExec
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
@@ -93,19 +75,16 @@ public class RestartCaseTest {
|
||||
assertThat(firstExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.getTaskRunList().getFirst().getAttempts().size() == 2,
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
executionQueue.emit(restartedExec);
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
}),
|
||||
Duration.ofSeconds(60)
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.getTaskRunList().getFirst().getAttempts().size() == 2,
|
||||
restartedExec
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
@@ -128,19 +107,16 @@ public class RestartCaseTest {
|
||||
assertThat(firstExecution.getTaskRunList().get(3).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.findTaskRunsByTaskId("failStep").stream().findFirst().get().getAttempts().size() == 2,
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
executionQueue.emit(restartedExec);
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(4);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
}),
|
||||
Duration.ofSeconds(60)
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(4);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
|
||||
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.findTaskRunsByTaskId("failStep").stream().findFirst().get().getAttempts().size() == 2,
|
||||
restartedExec
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
@@ -163,21 +139,19 @@ public class RestartCaseTest {
|
||||
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// wait
|
||||
Execution restartedExec = executionService.replay(firstExecution, firstExecution.findTaskRunByTaskIdAndValue("2_end", List.of()).getId(), null);
|
||||
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restartedExec.getState().getHistories()).hasSize(4);
|
||||
assertThat(restartedExec.getTaskRunList()).hasSize(20);
|
||||
assertThat(restartedExec.getTaskRunList().get(19).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
assertThat(restartedExec.getId()).isNotEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getTaskRunList().get(1).getId()).isNotEqualTo(firstExecution.getTaskRunList().get(1).getId());
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitChildExecution(
|
||||
flow,
|
||||
firstExecution,
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.replay(firstExecution, firstExecution.findTaskRunByTaskIdAndValue("2_end", List.of()).getId(), null);
|
||||
executionQueue.emit(restartedExec);
|
||||
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restartedExec.getState().getHistories()).hasSize(4);
|
||||
assertThat(restartedExec.getTaskRunList()).hasSize(20);
|
||||
assertThat(restartedExec.getTaskRunList().get(19).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
assertThat(restartedExec.getId()).isNotEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getTaskRunList().get(1).getId()).isNotEqualTo(firstExecution.getTaskRunList().get(1).getId());
|
||||
}),
|
||||
restartedExec,
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
@@ -195,71 +169,58 @@ public class RestartCaseTest {
|
||||
Execution restart = executionService.restart(execution, null);
|
||||
assertThat(restart.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
Execution restartEnded = runnerUtils.awaitExecution(
|
||||
Execution restartEnded = runnerUtils.emitAndAwaitExecution(
|
||||
e -> e.getState().getCurrent() == State.Type.FAILED,
|
||||
throwRunnable(() -> executionQueue.emit(restart)),
|
||||
Duration.ofSeconds(120)
|
||||
restart,
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(restartEnded.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
Execution newRestart = executionService.restart(restartEnded, null);
|
||||
|
||||
restartEnded = runnerUtils.awaitExecution(
|
||||
restartEnded = runnerUtils.emitAndAwaitExecution(
|
||||
e -> e.getState().getCurrent() == State.Type.FAILED,
|
||||
throwRunnable(() -> executionQueue.emit(newRestart)),
|
||||
Duration.ofSeconds(120)
|
||||
newRestart,
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(restartEnded.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
}
|
||||
|
||||
public void restartSubflow() throws Exception {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
Flux<Execution> receiveSubflows = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution subflowExecution = either.getLeft();
|
||||
if (subflowExecution.getFlowId().equals("restart-child") && subflowExecution.getState().getCurrent().isFailed()) {
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-parent");
|
||||
assertThat(execution.getTaskRunList()).hasSize(3);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// here we must have 1 failed subflows
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.MINUTES));
|
||||
receiveSubflows.blockLast();
|
||||
runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().isFailed(), MAIN_TENANT, "io.kestra.tests", "restart-child");
|
||||
|
||||
// there is 3 values so we must restart it 3 times to end the 3 subflows
|
||||
CountDownLatch successLatch = new CountDownLatch(3);
|
||||
receiveSubflows = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution subflowExecution = either.getLeft();
|
||||
if (subflowExecution.getFlowId().equals("restart-child") && subflowExecution.getState().getCurrent().isSuccess()) {
|
||||
successLatch.countDown();
|
||||
}
|
||||
});
|
||||
Execution restarted1 = executionService.restart(execution, null);
|
||||
execution = runnerUtils.awaitExecution(
|
||||
execution = runnerUtils.emitAndAwaitExecution(
|
||||
e -> e.getState().getCurrent() == State.Type.FAILED && e.getFlowId().equals("restart-parent"),
|
||||
throwRunnable(() -> executionQueue.emit(restarted1)),
|
||||
restarted1,
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Execution restarted2 = executionService.restart(execution, null);
|
||||
execution = runnerUtils.awaitExecution(
|
||||
execution = runnerUtils.emitAndAwaitExecution(
|
||||
e -> e.getState().getCurrent() == State.Type.FAILED && e.getFlowId().equals("restart-parent"),
|
||||
throwRunnable(() -> executionQueue.emit(restarted2)),
|
||||
restarted2,
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Execution restarted3 = executionService.restart(execution, null);
|
||||
execution = runnerUtils.awaitExecution(
|
||||
execution = runnerUtils.emitAndAwaitExecution(
|
||||
e -> e.getState().getCurrent() == State.Type.SUCCESS && e.getFlowId().equals("restart-parent"),
|
||||
throwRunnable(() -> executionQueue.emit(restarted3)),
|
||||
restarted3,
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
assertThat(execution.getTaskRunList()).hasSize(6);
|
||||
assertTrue(successLatch.await(1, TimeUnit.MINUTES));
|
||||
receiveSubflows.blockLast();
|
||||
|
||||
List<Execution> childExecutions = runnerUtils.awaitFlowExecutionNumber(3, MAIN_TENANT, "io.kestra.tests", "restart-child");
|
||||
List<Execution> successfulRestart = childExecutions.stream()
|
||||
.filter(e -> e.getState().getCurrent().equals(Type.SUCCESS)).toList();
|
||||
assertThat(successfulRestart).hasSize(3);
|
||||
}
|
||||
|
||||
public void restartFailedWithFinally() throws Exception {
|
||||
@@ -272,18 +233,15 @@ public class RestartCaseTest {
|
||||
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess() && execution.getId().equals(firstExecution.getId()),
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
executionQueue.emit(restartedExec);
|
||||
}),
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
|
||||
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess(),
|
||||
restartedExec,
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
@@ -309,21 +267,18 @@ public class RestartCaseTest {
|
||||
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait
|
||||
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
|
||||
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess() && execution.getId().equals(firstExecution.getId()),
|
||||
throwRunnable(() -> {
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
Execution restartedExec = executionService.restart(firstExecution, null);
|
||||
assertThat(restartedExec).isNotNull();
|
||||
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(restartedExec.getParentId()).isNull();
|
||||
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
|
||||
executionQueue.emit(restartedExec);
|
||||
}),
|
||||
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
|
||||
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess(),
|
||||
restartedExec,
|
||||
Duration.ofSeconds(60)
|
||||
);
|
||||
|
||||
assertThat(finishedRestartedExecution).isNotNull();
|
||||
assertThat(finishedRestartedExecution.getId()).isEqualTo(firstExecution.getId());
|
||||
assertThat(finishedRestartedExecution.getParentId()).isNull();
|
||||
|
||||
@@ -11,6 +11,7 @@ import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import org.junit.jupiter.api.RepeatedTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.event.Level;
|
||||
import reactor.core.publisher.Flux;
|
||||
@@ -25,6 +26,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
class RunContextLoggerTest {
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
|
||||
|
||||
@@ -98,7 +98,7 @@ class RunContextTest {
|
||||
private FlowInputOutput flowIO;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
protected LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@@ -1,13 +1,24 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.DependsOn;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.Type;
|
||||
import io.kestra.core.models.flows.input.BoolInput;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.property.PropertyContext;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.runners.pebble.functions.SecretFunction;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -112,4 +123,25 @@ class RunVariablesTest {
|
||||
assertThat(kestra.get("environment")).isEqualTo("test");
|
||||
assertThat(kestra.get("url")).isEqualTo("http://localhost:8080");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonResolvableDynamicInputsShouldBeSkipped() throws IllegalVariableEvaluationException {
|
||||
Map<String, Object> variables = new RunVariables.DefaultBuilder()
|
||||
.withFlow(Flow
|
||||
.builder()
|
||||
.namespace("a.b")
|
||||
.id("c")
|
||||
.inputs(List.of(
|
||||
BoolInput.builder().id("a").type(Type.BOOL).defaults(Property.ofValue(true)).build(),
|
||||
BoolInput.builder().id("b").type(Type.BOOL).dependsOn(new DependsOn(List.of("a"), null)).defaults(Property.ofExpression("{{inputs.a == true}}")).build()
|
||||
))
|
||||
.build()
|
||||
)
|
||||
.withExecution(Execution.builder().id(IdUtils.create()).build())
|
||||
.build(new RunContextLogger(), PropertyContext.create(new VariableRenderer(Mockito.mock(ApplicationContext.class), Mockito.mock(VariableRenderer.VariableConfiguration.class), Collections.emptyList())));
|
||||
|
||||
Assertions.assertEquals(Map.of(
|
||||
"a", true
|
||||
), variables.get("inputs"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@Singleton
|
||||
public class SLATestCase {
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public void maxDurationSLAShouldFail() throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-max-duration-fail");
|
||||
@@ -36,14 +36,14 @@ public class SLATestCase {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void executionConditionSLAShouldCancel() throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "CANCEL"));
|
||||
public void executionConditionSLAShouldCancel(String tenantId) throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "CANCEL"));
|
||||
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
}
|
||||
|
||||
public void executionConditionSLAShouldLabel() throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "LABEL"));
|
||||
public void executionConditionSLAShouldLabel(String tenantId) throws QueueException, TimeoutException {
|
||||
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "LABEL"));
|
||||
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(execution.getLabels()).contains(new Label("sla", "violated"));
|
||||
|
||||
@@ -3,54 +3,31 @@ package io.kestra.core.runners;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class ScheduleDateCaseTest {
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public void shouldScheduleOnDate() throws QueueException, InterruptedException {
|
||||
public void shouldScheduleOnDate(String tenantId) throws QueueException {
|
||||
ZonedDateTime scheduleOn = ZonedDateTime.now().plusSeconds(1);
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "minimal").orElseThrow();
|
||||
Flow flow = flowRepository.findById(tenantId, "io.kestra.tests", "minimal").orElseThrow();
|
||||
Execution execution = Execution.newExecution(flow, null, null, Optional.of(scheduleOn));
|
||||
this.executionQueue.emit(execution);
|
||||
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(execution.getScheduleDate()).isEqualTo(scheduleOn.toInstant());
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution.getId())) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ public class SkipExecutionCaseTest {
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user