mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-26 05:00:31 -05:00
Compare commits
285 Commits
v1.0.2
...
fix-execut
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01af20ad6d | ||
|
|
fa5108a6e9 | ||
|
|
a566c527cc | ||
|
|
d0c3c1daa0 | ||
|
|
5542aeef01 | ||
|
|
cf706a670e | ||
|
|
f91f28f2a5 | ||
|
|
db49b03269 | ||
|
|
e96da8ee43 | ||
|
|
e501fcb8cb | ||
|
|
9a8e84d460 | ||
|
|
f2b290dc32 | ||
|
|
a1b5d1b8e8 | ||
|
|
f541d77f0d | ||
|
|
57ad7bdd96 | ||
|
|
6ee910a2ec | ||
|
|
289a159dfd | ||
|
|
7e24495f71 | ||
|
|
aff8fde79d | ||
|
|
104d1c859c | ||
|
|
e20da7d5be | ||
|
|
c0f4bda10e | ||
|
|
b608c5a3b8 | ||
|
|
69c12874e4 | ||
|
|
5c24308e71 | ||
|
|
8fee5fc172 | ||
|
|
f633766bb9 | ||
|
|
f38b57ca4b | ||
|
|
9c3bf2c26b | ||
|
|
3a4e683685 | ||
|
|
a50c2c9ceb | ||
|
|
e4c35806cf | ||
|
|
2431567ee0 | ||
|
|
f6a496fb74 | ||
|
|
612e17a510 | ||
|
|
2814f8f159 | ||
|
|
279442bc8e | ||
|
|
d671f04de2 | ||
|
|
b7262f8f01 | ||
|
|
af00ee94f9 | ||
|
|
b628c3a218 | ||
|
|
0ed8193314 | ||
|
|
513e302bb2 | ||
|
|
296fb2fb7a | ||
|
|
5d883e0850 | ||
|
|
8d31d5407c | ||
|
|
f0720412d9 | ||
|
|
cb3ff02057 | ||
|
|
06ec05026e | ||
|
|
6a0929a050 | ||
|
|
7f714c0ffb | ||
|
|
83b4d285b1 | ||
|
|
bc137f2895 | ||
|
|
04052f3cbf | ||
|
|
08875d5292 | ||
|
|
189ad6090a | ||
|
|
02c896c3c5 | ||
|
|
154754e19c | ||
|
|
f2c3489f70 | ||
|
|
b741f7e3e7 | ||
|
|
4d931df726 | ||
|
|
ff742bfdd4 | ||
|
|
f0451df46f | ||
|
|
63b8e62b3f | ||
|
|
d243ba65e9 | ||
|
|
8771aa86a6 | ||
|
|
8a1cf71b63 | ||
|
|
4b9de17824 | ||
|
|
9bc2a9f8f2 | ||
|
|
0fce77cc8d | ||
|
|
04779e60c3 | ||
|
|
615502c58b | ||
|
|
08ac558e46 | ||
|
|
b43fd14625 | ||
|
|
c5d1e5bd38 | ||
|
|
4d89c5a9e1 | ||
|
|
5154127643 | ||
|
|
d205159e7e | ||
|
|
4f05198ae3 | ||
|
|
799500d4d1 | ||
|
|
9066063037 | ||
|
|
3b0f231eb5 | ||
|
|
f5a0dcc024 | ||
|
|
5c079b8b6b | ||
|
|
343d6b4eb9 | ||
|
|
d34d547412 | ||
|
|
7a542a24e2 | ||
|
|
5b1db68752 | ||
|
|
5b07b643d3 | ||
|
|
0e059772e4 | ||
|
|
f72e294e54 | ||
|
|
98dd884149 | ||
|
|
26c4f080fd | ||
|
|
01293de91c | ||
|
|
892b69f10e | ||
|
|
6f70d4d275 | ||
|
|
b41d2e456f | ||
|
|
5ec08eda8c | ||
|
|
7ed6b883ff | ||
|
|
eb166c9321 | ||
|
|
57aad1b931 | ||
|
|
60fe5b5c76 | ||
|
|
98c69b53bb | ||
|
|
d5d38559b4 | ||
|
|
4273ddc4f6 | ||
|
|
980c573a30 | ||
|
|
27109015f9 | ||
|
|
eba7d4f375 | ||
|
|
655a1172ee | ||
|
|
6e49a85acd | ||
|
|
4515bad6bd | ||
|
|
226dbd30c9 | ||
|
|
6b0c190edc | ||
|
|
c64df40a36 | ||
|
|
8af22d1bb2 | ||
|
|
b294457953 | ||
|
|
02d9c589fb | ||
|
|
6340d1c72f | ||
|
|
f439bd53d7 | ||
|
|
e54e3d5308 | ||
|
|
d084f2cd26 | ||
|
|
015960c78e | ||
|
|
39a09ecb67 | ||
|
|
45ce878d65 | ||
|
|
3ee647b9a8 | ||
|
|
7a7cb006bf | ||
|
|
911e6d5705 | ||
|
|
bf1458dde7 | ||
|
|
bd31e0eebd | ||
|
|
de02e4dd70 | ||
|
|
ec235b91fc | ||
|
|
ff1efa9958 | ||
|
|
e43c8ce387 | ||
|
|
2bd4e82b42 | ||
|
|
e63d6d1d86 | ||
|
|
a9752e65f2 | ||
|
|
151c56f3de | ||
|
|
d562ce3e65 | ||
|
|
681386a05b | ||
|
|
51ddfaf155 | ||
|
|
caee0a293f | ||
|
|
ba92880fa3 | ||
|
|
36b27510fb | ||
|
|
da2907e096 | ||
|
|
9b40665e64 | ||
|
|
0d35b5b355 | ||
|
|
339eb79854 | ||
|
|
0ee753529b | ||
|
|
84668fdfb9 | ||
|
|
9802f046e8 | ||
|
|
848b4d6577 | ||
|
|
1159bc5eb9 | ||
|
|
9b7ef37d14 | ||
|
|
89dfd18658 | ||
|
|
13ed2252bc | ||
|
|
c73b103bb3 | ||
|
|
396a077942 | ||
|
|
68e6fa2a4c | ||
|
|
a18748b3b2 | ||
|
|
236fcff7b4 | ||
|
|
cbbd697732 | ||
|
|
6b84737651 | ||
|
|
6ee7ecbd6b | ||
|
|
504f925085 | ||
|
|
7d37d2be93 | ||
|
|
94751a3b21 | ||
|
|
ba83b91680 | ||
|
|
56f62fb89f | ||
|
|
a0efe4b1f3 | ||
|
|
9af6338ae5 | ||
|
|
d53b933bdf | ||
|
|
a35c2816c5 | ||
|
|
d88eb9974c | ||
|
|
af3d5a384a | ||
|
|
e9ad352ccf | ||
|
|
1a95b83fb7 | ||
|
|
095939ff7a | ||
|
|
094f523874 | ||
|
|
c7efb2514a | ||
|
|
887537d8c1 | ||
|
|
0630b741b9 | ||
|
|
d2b7e723e1 | ||
|
|
080ceadf37 | ||
|
|
a89d902bc2 | ||
|
|
e2ef7d412a | ||
|
|
54c667ec4b | ||
|
|
1c53758d33 | ||
|
|
d092556bc2 | ||
|
|
308106d532 | ||
|
|
8fe8f96278 | ||
|
|
a5cad6d87c | ||
|
|
199d67fbe2 | ||
|
|
558a2e3f01 | ||
|
|
e1d2c30e54 | ||
|
|
700c6de411 | ||
|
|
2b838a5012 | ||
|
|
617daa79db | ||
|
|
1791127acb | ||
|
|
7feb571fb3 | ||
|
|
a315bd0e1c | ||
|
|
e2ac1e7e98 | ||
|
|
c6f40eff52 | ||
|
|
ccd42f7a1a | ||
|
|
ef08c8ac30 | ||
|
|
7b527c85a9 | ||
|
|
d121867066 | ||
|
|
a084a9f6f0 | ||
|
|
f6fff11081 | ||
|
|
3d5015938f | ||
|
|
951c93cedb | ||
|
|
9c06b37989 | ||
|
|
a916a03fdd | ||
|
|
4e728da331 | ||
|
|
166a3932c9 | ||
|
|
0a21971bbf | ||
|
|
8c4d7c0f9e | ||
|
|
b709913071 | ||
|
|
5be401d23c | ||
|
|
bb9f4be8c2 | ||
|
|
01e8e46b77 | ||
|
|
d00f4b0768 | ||
|
|
279f59c874 | ||
|
|
d897509726 | ||
|
|
0d592342af | ||
|
|
fc690bf7cd | ||
|
|
0a1b919863 | ||
|
|
2f4e981a29 | ||
|
|
5e7739432e | ||
|
|
8aba863b8c | ||
|
|
7eaa43c50f | ||
|
|
267ff78bfe | ||
|
|
7272cfe01f | ||
|
|
91e2fdb2cc | ||
|
|
a236688be6 | ||
|
|
81763d40ae | ||
|
|
677efb6739 | ||
|
|
b35924fef1 | ||
|
|
9dd93294b6 | ||
|
|
fac6dfe9a0 | ||
|
|
3bf9764505 | ||
|
|
c35cea5d19 | ||
|
|
4d8e9479f1 | ||
|
|
3f24e8e838 | ||
|
|
7175fcb666 | ||
|
|
2ddfa13b1b | ||
|
|
ba2a5dfec8 | ||
|
|
f84441dac7 | ||
|
|
433b788e4a | ||
|
|
65c5fd6331 | ||
|
|
421ab40276 | ||
|
|
efb2779693 | ||
|
|
74d371c0ca | ||
|
|
90a7869020 | ||
|
|
d9ccb50b0f | ||
|
|
aea0b87ef8 | ||
|
|
9a144fc3fe | ||
|
|
ddd9cebc63 | ||
|
|
1bebbb9b73 | ||
|
|
8de4dc867e | ||
|
|
fc49694e76 | ||
|
|
152300abae | ||
|
|
1ff5dda4e1 | ||
|
|
84f9b8876d | ||
|
|
575955567f | ||
|
|
d6d2580b45 | ||
|
|
070e54b902 | ||
|
|
829ca4380f | ||
|
|
381c7a75ad | ||
|
|
1688c489a9 | ||
|
|
93ccbf5f9b | ||
|
|
ac1cb235e5 | ||
|
|
9d3d3642e8 | ||
|
|
3d306a885e | ||
|
|
ef193c5774 | ||
|
|
d0f46169f4 | ||
|
|
3005ab527c | ||
|
|
688e2af12b | ||
|
|
4c0a05f484 | ||
|
|
108f8fc2c7 | ||
|
|
8b81a37559 | ||
|
|
9222f97d63 | ||
|
|
43e3591417 | ||
|
|
438dc9ecf6 | ||
|
|
7292837c58 | ||
|
|
7fa93d7764 |
29
.github/actions/plugins-list/action.yml
vendored
29
.github/actions/plugins-list/action.yml
vendored
@@ -1,29 +0,0 @@
|
||||
name: 'Load Kestra Plugin List'
|
||||
description: 'Composite action to load list of plugins'
|
||||
inputs:
|
||||
plugin-version:
|
||||
description: "Kestra version"
|
||||
default: 'LATEST'
|
||||
required: true
|
||||
plugin-file:
|
||||
description: "File of the plugins"
|
||||
default: './.plugins'
|
||||
required: true
|
||||
outputs:
|
||||
plugins:
|
||||
description: "List of all Kestra plugins"
|
||||
value: ${{ steps.plugins.outputs.plugins }}
|
||||
repositories:
|
||||
description: "List of all Kestra repositories of plugins"
|
||||
value: ${{ steps.plugins.outputs.repositories }}
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Get Plugins List
|
||||
id: plugins
|
||||
shell: bash
|
||||
run: |
|
||||
PLUGINS=$([ -f ${{ inputs.plugin-file }} ] && cat ${{ inputs.plugin-file }} | grep "io\\.kestra\\." | sed -e '/#/s/^.//' | sed -e "s/LATEST/${{ inputs.plugin-version }}/g" | cut -d':' -f2- | xargs || echo '');
|
||||
REPOSITORIES=$([ -f ${{ inputs.plugin-file }} ] && cat ${{ inputs.plugin-file }} | grep "io\\.kestra\\." | sed -e '/#/s/^.//' | cut -d':' -f1 | uniq | sort | xargs || echo '')
|
||||
echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT
|
||||
echo "repositories=$REPOSITORIES" >> $GITHUB_OUTPUT
|
||||
20
.github/actions/setup-vars/action.yml
vendored
20
.github/actions/setup-vars/action.yml
vendored
@@ -1,20 +0,0 @@
|
||||
name: 'Setup vars'
|
||||
description: 'Composite action to setup common vars'
|
||||
outputs:
|
||||
tag:
|
||||
description: "Git tag"
|
||||
value: ${{ steps.vars.outputs.tag }}
|
||||
commit:
|
||||
description: "Git commit"
|
||||
value: ${{ steps.vars.outputs.commit }}
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# Setup vars
|
||||
- name: Set variables
|
||||
id: vars
|
||||
shell: bash
|
||||
run: |
|
||||
TAG=${GITHUB_REF#refs/*/}
|
||||
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
||||
echo "commit=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_OUTPUT
|
||||
15
.github/workflows/e2e-scheduling.yml
vendored
Normal file
15
.github/workflows/e2e-scheduling.yml
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
name: 'E2E tests scheduling'
|
||||
# 'New E2E tests implementation started by Roman. Based on playwright in npm UI project, tests Kestra OSS develop docker image. These tests are written from zero, lets make them unflaky from the start!.'
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 * * * *" # Every hour
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
jobs:
|
||||
e2e:
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-e2e-tests.yml@main
|
||||
86
.github/workflows/e2e.yml
vendored
86
.github/workflows/e2e.yml
vendored
@@ -1,86 +0,0 @@
|
||||
name: 'E2E tests revival'
|
||||
description: 'New E2E tests implementation started by Roman. Based on playwright in npm UI project, tests Kestra OSS develop docker image. These tests are written from zero, lets make them unflaky from the start!.'
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 * * * *" # Every hour
|
||||
workflow_call:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
jobs:
|
||||
check:
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
steps:
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Checkout kestra
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
path: kestra
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
|
||||
- name: Install Npm dependencies
|
||||
run: |
|
||||
cd kestra/ui
|
||||
npm i
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run E2E Tests
|
||||
run: |
|
||||
cd kestra
|
||||
sh build-and-start-e2e-tests.sh
|
||||
|
||||
- name: Upload Playwright Report as Github artifact
|
||||
# 'With this report, you can analyze locally the results of the tests. see https://playwright.dev/docs/ci-intro#html-report'
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-report
|
||||
path: kestra/ui/playwright-report/
|
||||
retention-days: 7
|
||||
# Allure check
|
||||
# TODO I don't know what it should do
|
||||
# - uses: rlespinasse/github-slug-action@v5
|
||||
# name: Allure - Generate slug variables
|
||||
#
|
||||
# - name: Allure - Publish report
|
||||
# uses: andrcuns/allure-publish-action@v2.9.0
|
||||
# if: always() && env.GOOGLE_SERVICE_ACCOUNT != ''
|
||||
# continue-on-error: true
|
||||
# env:
|
||||
# GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
# JAVA_HOME: /usr/lib/jvm/default-jvm/
|
||||
# with:
|
||||
# storageType: gcs
|
||||
# resultsGlob: "**/build/allure-results"
|
||||
# bucket: internal-kestra-host
|
||||
# baseUrl: "https://internal.dev.kestra.io"
|
||||
# prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }}
|
||||
# copyLatest: true
|
||||
# ignoreMissingResults: true
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Run Gradle Release
|
||||
run-name: "Releasing Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
name: Create new release branch
|
||||
run-name: "Create new release branch Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: ./.github/actions/plugins-list
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: ./.github/actions/plugins-list
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--yes \
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--dry-run \
|
||||
@@ -1,5 +1,5 @@
|
||||
name: Set Version and Tag
|
||||
run-name: "Set version and Tag Kestra to ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
name: Start release
|
||||
run-name: "Start release of Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -7,17 +7,26 @@ on:
|
||||
description: 'The release version (e.g., 0.21.1)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/heads/releases/v')
|
||||
steps:
|
||||
# Checks
|
||||
- name: Check Inputs
|
||||
- name: Parse and Check Inputs
|
||||
id: parse-and-check-inputs
|
||||
run: |
|
||||
CURRENT_BRANCH="${{ github.ref_name }}"
|
||||
if ! [[ "$CURRENT_BRANCH" == "develop" ]]; then
|
||||
echo "You can only run this workflow on develop, but you ran it on $CURRENT_BRANCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)(\.[0-9]+)(-rc[0-9])?(-SNAPSHOT)?$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$"
|
||||
exit 1
|
||||
@@ -25,13 +34,8 @@ jobs:
|
||||
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
RELEASE_BRANCH="refs/heads/releases/v${BASE_VERSION}.x"
|
||||
|
||||
CURRENT_BRANCH="$GITHUB_REF"
|
||||
if ! [[ "$CURRENT_BRANCH" == "$RELEASE_BRANCH" ]]; then
|
||||
echo "Invalid release branch. Expected $RELEASE_BRANCH, was $CURRENT_BRANCH"
|
||||
exit 1
|
||||
fi
|
||||
RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
echo "release_branch=${RELEASE_BRANCH}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Checkout
|
||||
- name: Checkout
|
||||
@@ -39,6 +43,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
ref: ${{ steps.parse-and-check-inputs.outputs.release_branch }}
|
||||
|
||||
# Configure
|
||||
- name: Git - Configure
|
||||
@@ -47,7 +52,7 @@ jobs:
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
- name: Start release by updating version and pushing a new tag
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
@@ -1,6 +1,11 @@
|
||||
name: Main Workflow
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- releases/*
|
||||
- develop
|
||||
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
skip-test:
|
||||
@@ -11,53 +16,58 @@ on:
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
plugin-version:
|
||||
description: "plugins version"
|
||||
required: false
|
||||
type: string
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- main
|
||||
- releases/*
|
||||
- develop
|
||||
tags:
|
||||
- v*
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-main
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Execute tests
|
||||
uses: ./.github/workflows/workflow-test.yml
|
||||
backend-tests:
|
||||
name: Backend tests
|
||||
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
|
||||
with:
|
||||
report-status: false
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-backend-tests.yml@main
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
|
||||
release:
|
||||
name: Release
|
||||
needs: [tests]
|
||||
if: "!failure() && !cancelled() && !startsWith(github.ref, 'refs/heads/releases')"
|
||||
uses: ./.github/workflows/workflow-release.yml
|
||||
frontend-tests:
|
||||
name: Frontend tests
|
||||
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-frontend-tests.yml@main
|
||||
secrets:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
publish-develop-docker:
|
||||
name: Publish Docker
|
||||
needs: [backend-tests, frontend-tests]
|
||||
if: "!failure() && !cancelled() && github.ref == 'refs/heads/develop'"
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-docker.yml@main
|
||||
with:
|
||||
plugin-version: ${{ inputs.plugin-version != '' && inputs.plugin-version || (github.ref == 'refs/heads/develop' && 'LATEST-SNAPSHOT' || 'LATEST') }}
|
||||
plugin-version: 'LATEST-SNAPSHOT'
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
|
||||
publish-develop-maven:
|
||||
name: Publish develop Maven
|
||||
needs: [ backend-tests, frontend-tests ]
|
||||
if: "!failure() && !cancelled() && github.ref == 'refs/heads/develop'"
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-maven.yml@main
|
||||
secrets:
|
||||
SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
|
||||
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- release
|
||||
needs: [publish-develop-docker, publish-develop-maven]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Trigger EE Workflow
|
||||
49
.github/workflows/pre-release.yml
vendored
Normal file
49
.github/workflows/pre-release.yml
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
name: Pre Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-artifacts:
|
||||
name: Build Artifacts
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-build-artifacts.yml@main
|
||||
|
||||
backend-tests:
|
||||
name: Backend tests
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-backend-tests.yml@main
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
|
||||
frontend-tests:
|
||||
name: Frontend tests
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-frontend-tests.yml@main
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
publish-maven:
|
||||
name: Publish Maven
|
||||
needs: [ backend-tests, frontend-tests ]
|
||||
if: "!failure() && !cancelled()"
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-maven.yml@main
|
||||
secrets:
|
||||
SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
|
||||
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
|
||||
|
||||
publish-github:
|
||||
name: Github Release
|
||||
needs: [build-artifacts, backend-tests, frontend-tests]
|
||||
if: "!failure() && !cancelled()"
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-github.yml@main
|
||||
secrets:
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
@@ -3,7 +3,7 @@ name: Pull Request - Delete Docker
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
|
||||
# TODO import a reusable one
|
||||
jobs:
|
||||
publish:
|
||||
name: Pull Request - Delete Docker
|
||||
17
.github/workflows/pull-request.yml
vendored
17
.github/workflows/pull-request.yml
vendored
@@ -2,17 +2,12 @@ name: Pull Request Workflow
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- develop
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}-pr
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ********************************************************************************************************************
|
||||
# File changes detection
|
||||
# ********************************************************************************************************************
|
||||
file-changes:
|
||||
if: ${{ github.event.pull_request.draft == false }}
|
||||
name: File changes detection
|
||||
@@ -33,14 +28,11 @@ jobs:
|
||||
- '!{ui,.github}/**'
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# ********************************************************************************************************************
|
||||
# Tests
|
||||
# ********************************************************************************************************************
|
||||
frontend:
|
||||
name: Frontend - Tests
|
||||
needs: [file-changes]
|
||||
if: "needs.file-changes.outputs.ui == 'true'"
|
||||
uses: ./.github/workflows/workflow-frontend-test.yml
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-frontend-tests.yml@main
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
@@ -49,7 +41,7 @@ jobs:
|
||||
name: Backend - Tests
|
||||
needs: file-changes
|
||||
if: "needs.file-changes.outputs.backend == 'true'"
|
||||
uses: ./.github/workflows/workflow-backend-test.yml
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-backend-tests.yml@main
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
@@ -58,5 +50,8 @@ jobs:
|
||||
|
||||
e2e-tests:
|
||||
name: E2E - Tests
|
||||
uses: ./.github/workflows/e2e.yml
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-e2e-tests.yml@main
|
||||
|
||||
generate-pull-request-docker-image:
|
||||
name: Generate PR docker image
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-pullrequest-publish-docker.yml@main
|
||||
|
||||
34
.github/workflows/release-docker.yml
vendored
Normal file
34
.github/workflows/release-docker.yml
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
name: Publish docker
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
retag-latest:
|
||||
description: 'Retag latest Docker images'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
retag-lts:
|
||||
description: 'Retag LTS Docker images'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
plugin-version:
|
||||
description: 'Plugin version'
|
||||
required: false
|
||||
type: string
|
||||
default: "LATEST"
|
||||
|
||||
jobs:
|
||||
publish-docker:
|
||||
name: Publish Docker
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-docker.yml@main
|
||||
with:
|
||||
plugin-version: ${{ inputs.plugin-version }}
|
||||
retag-latest: ${{ inputs.retag-latest }}
|
||||
retag-lts: ${{ inputs.retag-lts }}
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
74
.github/workflows/workflow-backend-test.yml
vendored
74
.github/workflows/workflow-backend-test.yml
vendored
@@ -1,74 +0,0 @@
|
||||
name: Backend - Tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN:
|
||||
description: "The GitHub Token."
|
||||
required: true
|
||||
CODECOV_TOKEN:
|
||||
description: 'Codecov Token'
|
||||
required: true
|
||||
SONAR_TOKEN:
|
||||
description: 'Sonar Token'
|
||||
required: true
|
||||
GOOGLE_SERVICE_ACCOUNT:
|
||||
description: 'Google Service Account'
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
checks: write
|
||||
actions: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Backend - Tests
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
name: Checkout - Current ref
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
|
||||
# Services
|
||||
- name: Setup - Start docker compose
|
||||
shell: bash
|
||||
run: docker compose -f docker-compose-ci.yml up -d
|
||||
|
||||
# Gradle check
|
||||
- name: Gradle - Build
|
||||
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
|
||||
env:
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
shell: bash
|
||||
run: |
|
||||
echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json
|
||||
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
|
||||
./gradlew check javadoc --parallel
|
||||
|
||||
- name: comment PR with test report
|
||||
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
run: npx --yes @kestra-io/kestra-devtools generateTestReportSummary --only-errors --ci $(pwd)
|
||||
|
||||
# Report Java
|
||||
- name: Report - Java
|
||||
uses: kestra-io/actions/composite/report-java@main
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
secrets: ${{ toJSON(secrets) }}
|
||||
80
.github/workflows/workflow-build-artifacts.yml
vendored
80
.github/workflows/workflow-build-artifacts.yml
vendored
@@ -1,80 +0,0 @@
|
||||
name: Build Artifacts
|
||||
|
||||
on:
|
||||
workflow_call: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build - Artifacts
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
docker-tag: ${{ steps.vars.outputs.tag }}
|
||||
docker-artifact-name: ${{ steps.vars.outputs.artifact }}
|
||||
plugins: ${{ steps.plugins.outputs.plugins }}
|
||||
env:
|
||||
PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }}
|
||||
steps:
|
||||
- name: Checkout - Current ref
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Npm
|
||||
- name: Setup - Npm install
|
||||
shell: bash
|
||||
working-directory: ui
|
||||
run: npm ci
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
name: Setup - Build
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
|
||||
# Get Plugins List
|
||||
- name: Plugins - Get List
|
||||
uses: ./.github/actions/plugins-list
|
||||
if: "!startsWith(github.ref, 'refs/tags/v')"
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: ${{ env.PLUGIN_VERSION }}
|
||||
|
||||
# Set Plugins List
|
||||
- name: Plugins - Set List
|
||||
id: plugins
|
||||
if: "!startsWith(github.ref, 'refs/tags/v')"
|
||||
shell: bash
|
||||
run: |
|
||||
PLUGINS="${{ steps.plugins-list.outputs.plugins }}"
|
||||
TAG=${GITHUB_REF#refs/*/}
|
||||
if [[ $TAG = "master" || $TAG == v* ]]; then
|
||||
echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "plugins=--repositories=https://central.sonatype.com/repository/maven-snapshots/ $PLUGINS" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# Build
|
||||
- name: Gradle - Build
|
||||
shell: bash
|
||||
run: |
|
||||
./gradlew executableJar
|
||||
|
||||
- name: Artifacts - Copy exe to image
|
||||
shell: bash
|
||||
run: |
|
||||
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
|
||||
|
||||
# Upload artifacts
|
||||
- name: Artifacts - Upload JAR
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: jar
|
||||
path: build/libs/
|
||||
|
||||
- name: Artifacts - Upload Executable
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable/
|
||||
70
.github/workflows/workflow-frontend-test.yml
vendored
70
.github/workflows/workflow-frontend-test.yml
vendored
@@ -1,70 +0,0 @@
|
||||
name: Frontend - Tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN:
|
||||
description: "The GitHub Token."
|
||||
required: true
|
||||
CODECOV_TOKEN:
|
||||
description: 'Codecov Token'
|
||||
required: true
|
||||
|
||||
env:
|
||||
# to save corepack from itself
|
||||
COREPACK_INTEGRITY_KEYS: 0
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Frontend - Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Cache Node Modules
|
||||
id: cache-node-modules
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
ui/node_modules
|
||||
key: modules-${{ hashFiles('ui/package-lock.json') }}
|
||||
|
||||
- name: Cache Playwright Binaries
|
||||
id: cache-playwright
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: playwright-${{ hashFiles('ui/package-lock.json') }}
|
||||
|
||||
- name: Npm - install
|
||||
if: steps.cache-node-modules.outputs.cache-hit != 'true'
|
||||
working-directory: ui
|
||||
run: npm ci
|
||||
|
||||
- name: Npm - lint
|
||||
uses: reviewdog/action-eslint@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_AUTH_TOKEN }}
|
||||
reporter: github-pr-review
|
||||
workdir: ui
|
||||
|
||||
- name: Npm - Run build
|
||||
working-directory: ui
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
run: npm run build
|
||||
|
||||
- name: Run front-end unit tests
|
||||
working-directory: ui
|
||||
run: npm run test:unit -- --coverage
|
||||
|
||||
- name: Storybook - Install Playwright
|
||||
working-directory: ui
|
||||
if: steps.cache-playwright.outputs.cache-hit != 'true'
|
||||
run: npx playwright install --with-deps
|
||||
|
||||
- name: Run storybook component tests
|
||||
working-directory: ui
|
||||
run: npm run test:storybook -- --coverage
|
||||
79
.github/workflows/workflow-github-release.yml
vendored
79
.github/workflows/workflow-github-release.yml
vendored
@@ -1,79 +0,0 @@
|
||||
name: Github - Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
secrets:
|
||||
GH_PERSONAL_TOKEN:
|
||||
description: "The Github personal token."
|
||||
required: true
|
||||
SLACK_RELEASES_WEBHOOK_URL:
|
||||
description: "The Slack webhook URL."
|
||||
required: true
|
||||
|
||||
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Github - Release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Check out
|
||||
- name: Checkout - Repository
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
|
||||
|
||||
# Download Exec
|
||||
# Must be done after checkout actions
|
||||
- name: Artifacts - Download executable
|
||||
uses: actions/download-artifact@v5
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable
|
||||
|
||||
- name: Check if current tag is latest
|
||||
id: is_latest
|
||||
run: |
|
||||
latest_tag=$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sed 's/^v//' | sort -V | tail -n1)
|
||||
current_tag="${GITHUB_REF_NAME#v}"
|
||||
if [ "$current_tag" = "$latest_tag" ]; then
|
||||
echo "latest=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "latest=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
env:
|
||||
GITHUB_REF_NAME: ${{ github.ref_name }}
|
||||
|
||||
# GitHub Release
|
||||
- name: Create GitHub release
|
||||
uses: kestra-io/actions/composite/github-release@main
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
env:
|
||||
MAKE_LATEST: ${{ steps.is_latest.outputs.latest }}
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
|
||||
# Trigger gha workflow to bump helm chart version
|
||||
- name: GitHub - Trigger the Helm chart version bump
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
with:
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
repository: kestra-io/helm-charts
|
||||
event-type: update-helm-chart-version
|
||||
client-payload: |-
|
||||
{
|
||||
"new_version": "${{ github.ref_name }}",
|
||||
"github_repository": "${{ github.repository }}",
|
||||
"github_actor": "${{ github.actor }}"
|
||||
}
|
||||
|
||||
- name: Merge Release Notes
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
uses: kestra-io/actions/composite/github-release-note-merge@main
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
208
.github/workflows/workflow-publish-docker.yml
vendored
208
.github/workflows/workflow-publish-docker.yml
vendored
@@ -1,208 +0,0 @@
|
||||
name: Create Docker images on Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
retag-latest:
|
||||
description: 'Retag latest Docker images'
|
||||
required: true
|
||||
type: choice
|
||||
default: "false"
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
retag-lts:
|
||||
description: 'Retag LTS Docker images'
|
||||
required: true
|
||||
type: choice
|
||||
default: "false"
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
release-tag:
|
||||
description: 'Kestra Release Tag (by default, deduced with the ref)'
|
||||
required: false
|
||||
type: string
|
||||
plugin-version:
|
||||
description: 'Plugin version'
|
||||
required: false
|
||||
type: string
|
||||
default: "LATEST"
|
||||
force-download-artifact:
|
||||
description: 'Force download artifact'
|
||||
required: false
|
||||
type: choice
|
||||
default: "true"
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
workflow_call:
|
||||
inputs:
|
||||
plugin-version:
|
||||
description: "Plugin version"
|
||||
default: 'LATEST'
|
||||
required: false
|
||||
type: string
|
||||
force-download-artifact:
|
||||
description: 'Force download artifact'
|
||||
required: false
|
||||
type: string
|
||||
default: "true"
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME:
|
||||
description: "The Dockerhub username."
|
||||
required: true
|
||||
DOCKERHUB_PASSWORD:
|
||||
description: "The Dockerhub password."
|
||||
required: true
|
||||
|
||||
env:
|
||||
PLUGIN_VERSION: ${{ inputs.plugin-version != null && inputs.plugin-version || 'LATEST' }}
|
||||
jobs:
|
||||
plugins:
|
||||
name: List Plugins
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
plugins: ${{ steps.plugins.outputs.plugins }}
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: ./.github/actions/plugins-list
|
||||
id: plugins
|
||||
with: # remap LATEST-SNAPSHOT to LATEST
|
||||
plugin-version: ${{ env.PLUGIN_VERSION == 'LATEST-SNAPSHOT' && 'LATEST' || env.PLUGIN_VERSION }}
|
||||
|
||||
# ********************************************************************************************************************
|
||||
# Build
|
||||
# ********************************************************************************************************************
|
||||
build-artifacts:
|
||||
name: Build Artifacts
|
||||
if: ${{ inputs.force-download-artifact == 'true' }}
|
||||
uses: ./.github/workflows/workflow-build-artifacts.yml
|
||||
|
||||
docker:
|
||||
name: Publish Docker
|
||||
needs: [ plugins, build-artifacts ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
image:
|
||||
- name: "-no-plugins"
|
||||
plugins: ""
|
||||
packages: jattach
|
||||
python-libs: ""
|
||||
- name: ""
|
||||
plugins: ${{needs.plugins.outputs.plugins}}
|
||||
packages: python3 python-is-python3 python3-pip curl jattach
|
||||
python-libs: kestra
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
# Vars
|
||||
- name: Set image name
|
||||
id: vars
|
||||
run: |
|
||||
if [[ "${{ inputs.release-tag }}" == "" ]]; then
|
||||
TAG=${GITHUB_REF#refs/*/}
|
||||
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
TAG="${{ inputs.release-tag }}"
|
||||
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
if [[ $GITHUB_REF == refs/tags/* ]]; then
|
||||
if [[ $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
# this will remove the patch version number
|
||||
MINOR_SEMVER=${TAG%.*}
|
||||
echo "minor_semver=${MINOR_SEMVER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "Tag '$TAG' is not a valid semver (vMAJOR.MINOR.PATCH), skipping minor_semver"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${{ env.PLUGIN_VERSION }}" == *"-SNAPSHOT" ]]; then
|
||||
echo "plugins=--repositories=https://central.sonatype.com/repository/maven-snapshots/ ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT;
|
||||
else
|
||||
echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# Download executable from artifact
|
||||
- name: Artifacts - Download executable
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable
|
||||
|
||||
- name: Copy exe to image
|
||||
run: |
|
||||
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
|
||||
|
||||
# Docker setup
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Docker - Fix Qemu
|
||||
shell: bash
|
||||
run: |
|
||||
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Docker Login
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
|
||||
# Docker Build and push
|
||||
- name: Push to Docker Hub
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
build-args: |
|
||||
KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
|
||||
APT_PACKAGES=${{ matrix.image.packages }}
|
||||
PYTHON_LIBRARIES=${{ matrix.image.python-libs }}
|
||||
|
||||
- name: Install regctl
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: regclient/actions/regctl-installer@main
|
||||
|
||||
- name: Retag to minor semver version
|
||||
if: startsWith(github.ref, 'refs/tags/v') && steps.vars.outputs.minor_semver != ''
|
||||
run: |
|
||||
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.minor_semver, matrix.image.name) }}
|
||||
|
||||
- name: Retag to latest
|
||||
if: startsWith(github.ref, 'refs/tags/v') && inputs.retag-latest == 'true'
|
||||
run: |
|
||||
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest{0}', matrix.image.name) }}
|
||||
|
||||
- name: Retag to LTS
|
||||
if: startsWith(github.ref, 'refs/tags/v') && inputs.retag-lts == 'true'
|
||||
run: |
|
||||
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest-lts{0}', matrix.image.name) }}
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- docker
|
||||
if: always()
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
steps:
|
||||
- name: Slack notification
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
57
.github/workflows/workflow-publish-maven.yml
vendored
57
.github/workflows/workflow-publish-maven.yml
vendored
@@ -1,57 +0,0 @@
|
||||
name: Publish - Maven
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
secrets:
|
||||
SONATYPE_USER:
|
||||
description: "The Sonatype username."
|
||||
required: true
|
||||
SONATYPE_PASSWORD:
|
||||
description: "The Sonatype password."
|
||||
required: true
|
||||
SONATYPE_GPG_KEYID:
|
||||
description: "The Sonatype GPG key id."
|
||||
required: true
|
||||
SONATYPE_GPG_PASSWORD:
|
||||
description: "The Sonatype GPG password."
|
||||
required: true
|
||||
SONATYPE_GPG_FILE:
|
||||
description: "The Sonatype GPG file."
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish - Maven
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout - Current ref
|
||||
uses: actions/checkout@v5
|
||||
|
||||
# Setup build
|
||||
- name: Setup - Build
|
||||
uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
|
||||
# Publish
|
||||
- name: Publish - Release package to Maven Central
|
||||
shell: bash
|
||||
env:
|
||||
ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.SONATYPE_USER }}
|
||||
ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE}}
|
||||
run: |
|
||||
mkdir -p ~/.gradle/
|
||||
echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
|
||||
echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
|
||||
echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
|
||||
echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
|
||||
./gradlew publishToMavenCentral
|
||||
|
||||
# Gradle dependency
|
||||
- name: Java - Gradle dependency graph
|
||||
uses: gradle/actions/dependency-submission@v4
|
||||
@@ -1,78 +0,0 @@
|
||||
name: Pull Request - Publish Docker
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- develop
|
||||
|
||||
jobs:
|
||||
build-artifacts:
|
||||
name: Build Artifacts
|
||||
if: github.repository == 'kestra-io/kestra' # prevent running on forks
|
||||
uses: ./.github/workflows/workflow-build-artifacts.yml
|
||||
|
||||
publish:
|
||||
name: Publish Docker
|
||||
if: github.repository == 'kestra-io/kestra' # prevent running on forks
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-artifacts
|
||||
env:
|
||||
GITHUB_IMAGE_PATH: "ghcr.io/kestra-io/kestra-pr"
|
||||
steps:
|
||||
- name: Checkout - Current ref
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Docker setup
|
||||
- name: Docker - Setup QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Docker - Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# Docker Login
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Build Docker Image
|
||||
- name: Artifacts - Download executable
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
name: exe
|
||||
path: build/executable
|
||||
|
||||
- name: Docker - Copy exe to image
|
||||
shell: bash
|
||||
run: |
|
||||
cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
|
||||
|
||||
- name: Docker - Build image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.pr
|
||||
push: true
|
||||
tags: ${{ env.GITHUB_IMAGE_PATH }}:${{ github.event.pull_request.number }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
# Add comment on pull request
|
||||
- name: Add comment to PR
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: `**🐋 Docker image**: \`${{ env.GITHUB_IMAGE_PATH }}:${{ github.event.pull_request.number }}\`\n` +
|
||||
`\n` +
|
||||
`\`\`\`bash\n` +
|
||||
`docker run --pull=always --rm -it -p 8080:8080 --user=root -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp ${{ env.GITHUB_IMAGE_PATH }}:${{ github.event.pull_request.number }} server local\n` +
|
||||
`\`\`\``
|
||||
})
|
||||
85
.github/workflows/workflow-release.yml
vendored
85
.github/workflows/workflow-release.yml
vendored
@@ -1,85 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
plugin-version:
|
||||
description: "plugins version"
|
||||
default: 'LATEST'
|
||||
required: false
|
||||
type: string
|
||||
publish-docker:
|
||||
description: "Publish Docker image"
|
||||
default: 'false'
|
||||
required: false
|
||||
type: string
|
||||
workflow_call:
|
||||
inputs:
|
||||
plugin-version:
|
||||
description: "plugins version"
|
||||
default: 'LATEST'
|
||||
required: false
|
||||
type: string
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME:
|
||||
description: "The Dockerhub username."
|
||||
required: true
|
||||
DOCKERHUB_PASSWORD:
|
||||
description: "The Dockerhub password."
|
||||
required: true
|
||||
SONATYPE_USER:
|
||||
description: "The Sonatype username."
|
||||
required: true
|
||||
SONATYPE_PASSWORD:
|
||||
description: "The Sonatype password."
|
||||
required: true
|
||||
SONATYPE_GPG_KEYID:
|
||||
description: "The Sonatype GPG key id."
|
||||
required: true
|
||||
SONATYPE_GPG_PASSWORD:
|
||||
description: "The Sonatype GPG password."
|
||||
required: true
|
||||
SONATYPE_GPG_FILE:
|
||||
description: "The Sonatype GPG file."
|
||||
required: true
|
||||
GH_PERSONAL_TOKEN:
|
||||
description: "GH personnal Token."
|
||||
required: true
|
||||
SLACK_RELEASES_WEBHOOK_URL:
|
||||
description: "Slack webhook for releases channel."
|
||||
required: true
|
||||
jobs:
|
||||
build-artifacts:
|
||||
name: Build - Artifacts
|
||||
uses: ./.github/workflows/workflow-build-artifacts.yml
|
||||
|
||||
Docker:
|
||||
name: Publish Docker
|
||||
needs: build-artifacts
|
||||
uses: ./.github/workflows/workflow-publish-docker.yml
|
||||
if: github.ref == 'refs/heads/develop' || inputs.publish-docker == 'true'
|
||||
with:
|
||||
force-download-artifact: 'false'
|
||||
plugin-version: ${{ inputs.plugin-version != null && inputs.plugin-version || 'LATEST' }}
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
|
||||
Maven:
|
||||
name: Publish Maven
|
||||
uses: ./.github/workflows/workflow-publish-maven.yml
|
||||
secrets:
|
||||
SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
|
||||
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
|
||||
SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
|
||||
SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
|
||||
SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
|
||||
|
||||
Github:
|
||||
name: Github Release
|
||||
needs: build-artifacts
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: ./.github/workflows/workflow-github-release.yml
|
||||
secrets:
|
||||
GH_PERSONAL_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
SLACK_RELEASES_WEBHOOK_URL: ${{ secrets.SLACK_RELEASES_WEBHOOK_URL }}
|
||||
95
.github/workflows/workflow-test.yml
vendored
95
.github/workflows/workflow-test.yml
vendored
@@ -1,95 +0,0 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * 1,2,3,4,5'
|
||||
workflow_call:
|
||||
inputs:
|
||||
report-status:
|
||||
description: "Report status of the jobs in outputs"
|
||||
type: string
|
||||
required: false
|
||||
default: false
|
||||
outputs:
|
||||
frontend_status:
|
||||
description: "Status of the frontend job"
|
||||
value: ${{ jobs.set-frontend-status.outputs.frontend_status }}
|
||||
backend_status:
|
||||
description: "Status of the backend job"
|
||||
value: ${{ jobs.set-backend-status.outputs.backend_status }}
|
||||
|
||||
jobs:
|
||||
file-changes:
|
||||
name: File changes detection
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
ui: ${{ steps.changes.outputs.ui }}
|
||||
backend: ${{ steps.changes.outputs.backend }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
if: "!startsWith(github.ref, 'refs/tags/v')"
|
||||
- uses: dorny/paths-filter@v3
|
||||
if: "!startsWith(github.ref, 'refs/tags/v')"
|
||||
id: changes
|
||||
with:
|
||||
filters: |
|
||||
ui:
|
||||
- 'ui/**'
|
||||
backend:
|
||||
- '!{ui,.github}/**'
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
frontend:
|
||||
name: Frontend - Tests
|
||||
needs: file-changes
|
||||
if: "needs.file-changes.outputs.ui == 'true' || startsWith(github.ref, 'refs/tags/v')"
|
||||
uses: ./.github/workflows/workflow-frontend-test.yml
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
|
||||
backend:
|
||||
name: Backend - Tests
|
||||
needs: file-changes
|
||||
if: "needs.file-changes.outputs.backend == 'true' || startsWith(github.ref, 'refs/tags/v')"
|
||||
uses: ./.github/workflows/workflow-backend-test.yml
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
|
||||
|
||||
# Output every job status
|
||||
# To be used in other workflows
|
||||
report-status:
|
||||
name: Report Status
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ frontend, backend ]
|
||||
if: always() && (inputs.report-status == 'true')
|
||||
outputs:
|
||||
frontend_status: ${{ steps.set-frontend-status.outputs.frontend_status }}
|
||||
backend_status: ${{ steps.set-backend-status.outputs.backend_status }}
|
||||
steps:
|
||||
- id: set-frontend-status
|
||||
name: Set frontend job status
|
||||
run: echo "::set-output name=frontend_status::${{ needs.frontend.result }}"
|
||||
|
||||
- id: set-backend-status
|
||||
name: Set backend job status
|
||||
run: echo "::set-output name=backend_status::${{ needs.backend.result }}"
|
||||
|
||||
notify:
|
||||
name: Notify - Slack
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ frontend, backend ]
|
||||
steps:
|
||||
- name: Notify failed CI
|
||||
if: |
|
||||
always() &&
|
||||
(needs.frontend.result != 'success' || needs.backend.result != 'success') &&
|
||||
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
13
README.md
13
README.md
@@ -19,9 +19,12 @@
|
||||
<br />
|
||||
|
||||
<p align="center">
|
||||
<a href="https://x.com/kestra_io"><img height="25" src="https://kestra.io/twitter.svg" alt="X(formerly Twitter)" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/"><img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io"><img height="25" src="https://kestra.io/youtube.svg" alt="youtube" /></a>
|
||||
<a href="https://twitter.com/kestra_io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/twitter.svg" alt="twitter" width="35" height="25" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" width="35" height="25" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/youtube.svg" alt="youtube" width="35" height="25" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -33,10 +36,10 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.kestra.io/video/product-overview" target="_blank">
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
|
||||
</a>
|
||||
</p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
|
||||
|
||||
|
||||
## 🌟 What is Kestra?
|
||||
|
||||
27
build.gradle
27
build.gradle
@@ -25,19 +25,19 @@ plugins {
|
||||
id 'jacoco-report-aggregation'
|
||||
|
||||
// helper
|
||||
id "com.github.ben-manes.versions" version "0.52.0"
|
||||
id "com.github.ben-manes.versions" version "0.53.0"
|
||||
|
||||
// front
|
||||
id 'com.github.node-gradle.node' version '7.1.0'
|
||||
|
||||
// release
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.2"
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.3"
|
||||
id 'signing'
|
||||
id "com.vanniktech.maven.publish" version "0.34.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.3" apply false
|
||||
id "org.owasp.dependencycheck" version "12.1.5" apply false
|
||||
}
|
||||
|
||||
idea {
|
||||
@@ -168,8 +168,9 @@ allprojects {
|
||||
/**********************************************************************************************************************\
|
||||
* Test
|
||||
**********************************************************************************************************************/
|
||||
subprojects {
|
||||
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
|
||||
subprojects {subProj ->
|
||||
|
||||
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
|
||||
apply plugin: "com.adarshr.test-logger"
|
||||
|
||||
java {
|
||||
@@ -207,6 +208,13 @@ subprojects {
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
reports {
|
||||
junitXml.required = true
|
||||
junitXml.outputPerTestCase = true
|
||||
junitXml.mergeReruns = true
|
||||
junitXml.includeSystemErrLog = true;
|
||||
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
|
||||
}
|
||||
|
||||
// set Xmx for test workers
|
||||
maxHeapSize = '4g'
|
||||
@@ -222,6 +230,15 @@ subprojects {
|
||||
environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
|
||||
environment 'ENV_TEST1', "true"
|
||||
environment 'ENV_TEST2', "Pass by env"
|
||||
|
||||
|
||||
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
|
||||
// JUnit 5 parallel settings
|
||||
systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
|
||||
systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
|
||||
systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
|
||||
systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
|
||||
}
|
||||
}
|
||||
|
||||
testlogger {
|
||||
|
||||
@@ -40,5 +40,6 @@ dependencies {
|
||||
implementation project(":worker")
|
||||
|
||||
//test
|
||||
testImplementation project(':tests')
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
@@ -49,7 +49,7 @@ import java.util.concurrent.Callable;
|
||||
@Introspected
|
||||
public class App implements Callable<Integer> {
|
||||
public static void main(String[] args) {
|
||||
execute(App.class, args);
|
||||
execute(App.class, new String [] { Environment.CLI }, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
|
||||
return PicocliRunner.call(App.class, "--help");
|
||||
}
|
||||
|
||||
protected static void execute(Class<?> cls, String... args) {
|
||||
protected static void execute(Class<?> cls, String[] environments, String... args) {
|
||||
// Log Bridge
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
|
||||
// Init ApplicationContext
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, args);
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
|
||||
|
||||
// Call Picocli command
|
||||
int exitCode = 0;
|
||||
@@ -80,6 +80,7 @@ public class App implements Callable<Integer> {
|
||||
System.exit(Objects.requireNonNullElse(exitCode, 0));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
|
||||
* forced Properties from current command.
|
||||
@@ -88,12 +89,13 @@ public class App implements Callable<Integer> {
|
||||
* @return the application context created
|
||||
*/
|
||||
protected static ApplicationContext applicationContext(Class<?> mainClass,
|
||||
String[] environments,
|
||||
String[] args) {
|
||||
|
||||
ApplicationContextBuilder builder = ApplicationContext
|
||||
.builder()
|
||||
.mainClass(mainClass)
|
||||
.environments(Environment.CLI);
|
||||
.environments(environments);
|
||||
|
||||
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
|
||||
continueOnParsingErrors(cmd);
|
||||
|
||||
@@ -2,19 +2,27 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@Slf4j
|
||||
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
|
||||
Integer serverPort;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
|
||||
|
||||
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
|
||||
|
||||
return super.call();
|
||||
}
|
||||
|
||||
private long maxMemoryInMB() {
|
||||
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
|
||||
}
|
||||
|
||||
protected static int defaultWorkerThread() {
|
||||
return Runtime.getRuntime().availableProcessors() * 8;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package io.kestra.cli.commands.servers;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.models.ServerType;
|
||||
import io.kestra.core.runners.ExecutorInterface;
|
||||
import io.kestra.executor.SkipExecutionService;
|
||||
import io.kestra.core.services.SkipExecutionService;
|
||||
import io.kestra.core.services.StartExecutorService;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
|
||||
@@ -4,10 +4,13 @@ import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.models.ServerType;
|
||||
import io.kestra.core.runners.Indexer;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.services.SkipExecutionService;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import jakarta.inject.Inject;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@CommandLine.Command(
|
||||
@@ -17,6 +20,11 @@ import java.util.Map;
|
||||
public class IndexerCommand extends AbstractServerCommand {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
@Inject
|
||||
private SkipExecutionService skipExecutionService;
|
||||
|
||||
@CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
|
||||
private List<String> skipIndexerRecords = Collections.emptyList();
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public static Map<String, Object> propertiesOverrides() {
|
||||
@@ -27,6 +35,8 @@ public class IndexerCommand extends AbstractServerCommand {
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
this.skipExecutionService.setSkipIndexerRecords(skipIndexerRecords);
|
||||
|
||||
super.call();
|
||||
|
||||
Indexer indexer = applicationContext.getBean(Indexer.class);
|
||||
|
||||
@@ -7,7 +7,7 @@ import io.kestra.core.contexts.KestraContext;
|
||||
import io.kestra.core.models.ServerType;
|
||||
import io.kestra.core.repositories.LocalFlowRepositoryLoader;
|
||||
import io.kestra.cli.StandAloneRunner;
|
||||
import io.kestra.executor.SkipExecutionService;
|
||||
import io.kestra.core.services.SkipExecutionService;
|
||||
import io.kestra.core.services.StartExecutorService;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
@@ -63,6 +63,9 @@ public class StandAloneCommand extends AbstractServerCommand {
|
||||
@CommandLine.Option(names = {"--skip-tenants"}, split=",", description = "a list of tenants to skip, separated by a coma; for troubleshooting purpose only")
|
||||
private List<String> skipTenants = Collections.emptyList();
|
||||
|
||||
@CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
|
||||
private List<String> skipIndexerRecords = Collections.emptyList();
|
||||
|
||||
@CommandLine.Option(names = {"--no-tutorials"}, description = "Flag to disable auto-loading of tutorial flows.")
|
||||
boolean tutorialsDisabled = false;
|
||||
|
||||
@@ -93,6 +96,7 @@ public class StandAloneCommand extends AbstractServerCommand {
|
||||
this.skipExecutionService.setSkipFlows(skipFlows);
|
||||
this.skipExecutionService.setSkipNamespaces(skipNamespaces);
|
||||
this.skipExecutionService.setSkipTenants(skipTenants);
|
||||
this.skipExecutionService.setSkipIndexerRecords(skipIndexerRecords);
|
||||
this.startExecutorService.applyOptions(startExecutors, notStartExecutors);
|
||||
|
||||
KestraContext.getContext().injectWorkerConfigs(workerThread, null);
|
||||
|
||||
@@ -5,12 +5,15 @@ import io.kestra.core.models.ServerType;
|
||||
import io.kestra.core.runners.Indexer;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.ExecutorsUtils;
|
||||
import io.kestra.core.services.SkipExecutionService;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
@@ -28,11 +31,17 @@ public class WebServerCommand extends AbstractServerCommand {
|
||||
@Inject
|
||||
private ExecutorsUtils executorsUtils;
|
||||
|
||||
@Inject
|
||||
private SkipExecutionService skipExecutionService;
|
||||
|
||||
@Option(names = {"--no-tutorials"}, description = "Flag to disable auto-loading of tutorial flows.")
|
||||
boolean tutorialsDisabled = false;
|
||||
private boolean tutorialsDisabled = false;
|
||||
|
||||
@Option(names = {"--no-indexer"}, description = "Flag to disable starting an embedded indexer.")
|
||||
boolean indexerDisabled = false;
|
||||
private boolean indexerDisabled = false;
|
||||
|
||||
@CommandLine.Option(names = {"--skip-indexer-records"}, split=",", description = "a list of indexer record keys, separated by a coma; for troubleshooting purpose only")
|
||||
private List<String> skipIndexerRecords = Collections.emptyList();
|
||||
|
||||
@Override
|
||||
public boolean isFlowAutoLoadEnabled() {
|
||||
@@ -48,6 +57,8 @@ public class WebServerCommand extends AbstractServerCommand {
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
this.skipExecutionService.setSkipIndexerRecords(skipIndexerRecords);
|
||||
|
||||
super.call();
|
||||
|
||||
// start the indexer
|
||||
|
||||
@@ -262,6 +262,8 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private String getTenantIdFromPath(Path path) {
|
||||
// FIXME there is probably a bug here when a tenant has '_' in its name,
|
||||
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
|
||||
return path.getFileName().toString().split("_")[0];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +169,7 @@ kestra:
|
||||
- "/api/v1/executions/webhook/"
|
||||
- "/api/v1/main/executions/webhook/"
|
||||
- "/api/v1/*/executions/webhook/"
|
||||
- "/api/v1/basicAuthValidationErrors"
|
||||
|
||||
preview:
|
||||
initial-rows: 100
|
||||
|
||||
@@ -37,7 +37,7 @@ class AppTest {
|
||||
|
||||
final String[] args = new String[]{"server", serverType, "--help"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
|
||||
|
||||
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
|
||||
@@ -52,7 +52,7 @@ class AppTest {
|
||||
|
||||
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
|
||||
|
||||
assertThat(out.toString()).startsWith("Missing required parameters: ");
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package io.kestra.cli.services;
|
||||
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -18,8 +19,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -57,10 +58,12 @@ class FileChangedEventListenerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
void test() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a basic flow
|
||||
String flow = """
|
||||
@@ -73,14 +76,14 @@ class FileChangedEventListenerTest {
|
||||
message: Hello World! 🚀
|
||||
""";
|
||||
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
assertThat(myflow.getTasks()).hasSize(1);
|
||||
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
|
||||
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -88,16 +91,18 @@ class FileChangedEventListenerTest {
|
||||
// delete the flow
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
void testWithPluginDefault() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a flow with plugin default
|
||||
String pluginDefault = """
|
||||
@@ -113,14 +118,14 @@ class FileChangedEventListenerTest {
|
||||
values:
|
||||
message: Hello World!
|
||||
""";
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -128,7 +133,7 @@ class FileChangedEventListenerTest {
|
||||
// delete both files
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
|
||||
@@ -84,7 +84,7 @@ dependencies {
|
||||
|
||||
testImplementation "org.testcontainers:testcontainers:1.21.3"
|
||||
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on"
|
||||
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
|
||||
@@ -118,7 +118,7 @@ public class JsonSchemaGenerator {
|
||||
removeRequiredOnPropsWithDefaults(objectNode);
|
||||
|
||||
return MAPPER.convertValue(objectNode, MAP_TYPE_REFERENCE);
|
||||
} catch (IllegalArgumentException e) {
|
||||
} catch (Exception e) {
|
||||
throw new IllegalArgumentException("Unable to generate jsonschema for '" + cls.getName() + "'", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.utils.MapUtils;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public record Label(@NotNull String key, @NotNull String value) {
|
||||
@Schema(description = "A key/value pair that can be attached to a Flow or Execution. Labels are often used to organize and categorize objects.")
|
||||
public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static final String SYSTEM_PREFIX = "system.";
|
||||
|
||||
// system labels
|
||||
@@ -41,7 +44,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
@@ -56,6 +59,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
@@ -70,6 +74,7 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
if (map == null || map.isEmpty()) return List.of();
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
@@ -88,4 +93,14 @@ public record Label(@NotNull String key, @NotNull String value) {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides predicate for not empty entries.
|
||||
*
|
||||
* @return The non-empty filter
|
||||
*/
|
||||
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
|
||||
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
|
||||
entry.getValue() != null && !entry.getValue().isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,33 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface that can be implemented by classes supporting plugin versioning.
|
||||
*
|
||||
* @see Plugin
|
||||
*/
|
||||
public interface PluginVersioning {
|
||||
|
||||
String TITLE = "Plugin Version";
|
||||
String DESCRIPTION = """
|
||||
Defines the version of the plugin to use.
|
||||
|
||||
@Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)")
|
||||
@Schema(title = "The version of the plugin to use.")
|
||||
The version must follow the Semantic Versioning (SemVer) specification:
|
||||
- A single-digit MAJOR version (e.g., `1`).
|
||||
- A MAJOR.MINOR version (e.g., `1.1`).
|
||||
- A MAJOR.MINOR.PATCH version, optionally with any qualifier
|
||||
(e.g., `1.1.2`, `1.1.0-SNAPSHOT`).
|
||||
""";
|
||||
|
||||
@Schema(
|
||||
title = TITLE,
|
||||
description = DESCRIPTION
|
||||
)
|
||||
String getVersion();
|
||||
}
|
||||
|
||||
@@ -254,19 +254,7 @@ public record QueryFilter(
|
||||
*
|
||||
* @return List of {@code ResourceField} with resource names, fields, and operations.
|
||||
*/
|
||||
public static List<ResourceField> asResourceList() {
|
||||
return Arrays.stream(values())
|
||||
.map(Resource::toResourceField)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ResourceField toResourceField(Resource resource) {
|
||||
List<FieldOp> fieldOps = resource.supportedField().stream()
|
||||
.map(Resource::toFieldInfo)
|
||||
.toList();
|
||||
return new ResourceField(resource.name().toLowerCase(), fieldOps);
|
||||
}
|
||||
|
||||
|
||||
private static FieldOp toFieldInfo(Field field) {
|
||||
List<Operation> operations = field.supportedOp().stream()
|
||||
.map(Resource::toOperation)
|
||||
@@ -279,9 +267,6 @@ public record QueryFilter(
|
||||
}
|
||||
}
|
||||
|
||||
public record ResourceField(String name, List<FieldOp> fields) {
|
||||
}
|
||||
|
||||
public record FieldOp(String name, String value, List<Operation> operations) {
|
||||
}
|
||||
|
||||
|
||||
@@ -17,31 +17,12 @@ import java.util.List;
|
||||
@Introspected
|
||||
public class ExecutionUsage {
|
||||
private final List<DailyExecutionStatistics> dailyExecutionsCount;
|
||||
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
|
||||
|
||||
public static ExecutionUsage of(final String tenantId,
|
||||
final ExecutionRepositoryInterface executionRepository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
|
||||
try {
|
||||
dailyTaskRunsCount = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenantId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
true);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
|
||||
}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(executionRepository.dailyStatistics(
|
||||
null,
|
||||
@@ -52,28 +33,13 @@ public class ExecutionUsage {
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
false))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
null))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
try {
|
||||
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
true
|
||||
);
|
||||
} catch (UnsupportedOperationException ignored) {}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
@@ -81,10 +47,8 @@ public class ExecutionUsage {
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
false
|
||||
DateUtils.GroupType.DAY
|
||||
))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -272,7 +272,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
public Execution withTaskRun(TaskRun taskRun) throws InternalException {
|
||||
ArrayList<TaskRun> newTaskRunList = new ArrayList<>(this.taskRunList);
|
||||
ArrayList<TaskRun> newTaskRunList = this.taskRunList == null ? new ArrayList<>() : new ArrayList<>(this.taskRunList);
|
||||
|
||||
boolean b = Collections.replaceAll(
|
||||
newTaskRunList,
|
||||
@@ -865,20 +865,18 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
* @param e the exception raise
|
||||
* @return new taskRun with updated attempt with logs
|
||||
*/
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
|
||||
TaskRunAttempt lastAttempt, Exception e) {
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
|
||||
TaskRun failed = taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
return new FailedTaskRunWithLog(
|
||||
taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt
|
||||
.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
)
|
||||
.withState(State.Type.FAILED),
|
||||
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
|
||||
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -296,7 +296,7 @@ public class TaskRun implements TenantInterface {
|
||||
}
|
||||
|
||||
public TaskRun incrementIteration() {
|
||||
int iteration = this.iteration == null ? 1 : this.iteration;
|
||||
int iteration = this.iteration == null ? 0 : this.iteration;
|
||||
return this.toBuilder()
|
||||
.iteration(iteration + 1)
|
||||
.build();
|
||||
|
||||
@@ -3,7 +3,6 @@ package io.kestra.core.models.flows;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.annotations.PluginProperty;
|
||||
import io.kestra.core.models.tasks.WorkerGroup;
|
||||
import io.kestra.core.serializers.ListOrMapOfLabelDeserializer;
|
||||
import io.kestra.core.serializers.ListOrMapOfLabelSerializer;
|
||||
@@ -61,7 +60,14 @@ public abstract class AbstractFlow implements FlowInterface {
|
||||
|
||||
@JsonSerialize(using = ListOrMapOfLabelSerializer.class)
|
||||
@JsonDeserialize(using = ListOrMapOfLabelDeserializer.class)
|
||||
@Schema(implementation = Object.class, oneOf = {List.class, Map.class})
|
||||
@Schema(
|
||||
description = "Labels as a list of Label (key/value pairs) or as a map of string to string.",
|
||||
oneOf = {
|
||||
Label[].class,
|
||||
Map.class
|
||||
}
|
||||
)
|
||||
@Valid
|
||||
List<Label> labels;
|
||||
|
||||
@Schema(additionalProperties = Schema.AdditionalPropertiesValue.TRUE)
|
||||
@@ -69,4 +75,5 @@ public abstract class AbstractFlow implements FlowInterface {
|
||||
|
||||
@Valid
|
||||
private WorkerGroup workerGroup;
|
||||
|
||||
}
|
||||
|
||||
@@ -86,10 +86,11 @@ public class State {
|
||||
|
||||
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
|
||||
public Duration getDuration() {
|
||||
return Duration.between(
|
||||
this.histories.getFirst().getDate(),
|
||||
this.histories.size() > 1 ? this.histories.get(this.histories.size() - 1).getDate() : Instant.now()
|
||||
);
|
||||
if(this.getEndDate().isPresent()){
|
||||
return Duration.between(this.getStartDate(), this.getEndDate().get());
|
||||
} else {
|
||||
return Duration.between(this.getStartDate(), Instant.now());
|
||||
}
|
||||
}
|
||||
|
||||
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
|
||||
|
||||
@@ -30,7 +30,7 @@ public class ResolvedTask {
|
||||
|
||||
public NextTaskRun toNextTaskRunIncrementIteration(Execution execution, Integer iteration) {
|
||||
return new NextTaskRun(
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 1),
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 0),
|
||||
this.getTask()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,34 +185,6 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Trigger update(Trigger currentTrigger, Trigger newTrigger, ZonedDateTime nextExecutionDate) throws Exception {
|
||||
Trigger updated = currentTrigger;
|
||||
|
||||
// If a backfill is created, we update the currentTrigger
|
||||
// and set the nextExecutionDate() as the previous one
|
||||
if (newTrigger.getBackfill() != null) {
|
||||
updated = currentTrigger.toBuilder()
|
||||
.backfill(
|
||||
newTrigger
|
||||
.getBackfill()
|
||||
.toBuilder()
|
||||
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
|
||||
.currentDate(
|
||||
newTrigger.getBackfill().getStart()
|
||||
)
|
||||
.previousNextExecutionDate(
|
||||
currentTrigger.getNextExecutionDate())
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
return updated.toBuilder()
|
||||
.nextExecutionDate(newTrigger.getDisabled() ?
|
||||
null : nextExecutionDate)
|
||||
.disabled(newTrigger.getDisabled())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Trigger resetExecution(Flow flow, Execution execution, ConditionContext conditionContext) {
|
||||
boolean disabled = this.getStopAfter() != null ? this.getStopAfter().contains(execution.getState().getCurrent()) : this.getDisabled();
|
||||
if (!disabled) {
|
||||
@@ -276,27 +248,22 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
.build();
|
||||
}
|
||||
|
||||
public Trigger initBackfill(Trigger newTrigger) {
|
||||
// If a backfill is created, we update the currentTrigger
|
||||
public Trigger withBackfill(final Backfill backfill) {
|
||||
Trigger updated = this;
|
||||
// If a backfill is created, we update the trigger
|
||||
// and set the nextExecutionDate() as the previous one
|
||||
if (newTrigger.getBackfill() != null) {
|
||||
|
||||
return this.toBuilder()
|
||||
if (backfill != null) {
|
||||
updated = this.toBuilder()
|
||||
.backfill(
|
||||
newTrigger
|
||||
.getBackfill()
|
||||
backfill
|
||||
.toBuilder()
|
||||
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
|
||||
.currentDate(
|
||||
newTrigger.getBackfill().getStart()
|
||||
)
|
||||
.previousNextExecutionDate(
|
||||
this.getNextExecutionDate())
|
||||
.end(backfill.getEnd() != null ? backfill.getEnd() : ZonedDateTime.now())
|
||||
.currentDate(backfill.getStart())
|
||||
.previousNextExecutionDate(this.getNextExecutionDate())
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
return this;
|
||||
return updated;
|
||||
}
|
||||
|
||||
// if the next date is after the backfill end, we remove the backfill
|
||||
|
||||
@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
*
|
||||
* @return the {@link DefaultPluginRegistry}.
|
||||
*/
|
||||
public static DefaultPluginRegistry getOrCreate() {
|
||||
public synchronized static DefaultPluginRegistry getOrCreate() {
|
||||
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
|
||||
if (!instance.isInitialized()) {
|
||||
instance.init();
|
||||
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
/**
|
||||
* Initializes the registry by loading all core plugins.
|
||||
*/
|
||||
protected void init() {
|
||||
protected synchronized void init() {
|
||||
if (initialized.compareAndSet(false, true)) {
|
||||
register(scanner.scan());
|
||||
}
|
||||
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
if (existing != null && existing.crc32() == plugin.crc32()) {
|
||||
return; // same plugin already registered
|
||||
}
|
||||
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
if (existing != null) {
|
||||
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
|
||||
pluginClassByIdentifier.putAll(plugins);
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
|
||||
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
|
||||
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::asText).orElse(null);
|
||||
|
||||
if (type == null || type.isEmpty()) {
|
||||
return null;
|
||||
|
||||
@@ -27,6 +27,7 @@ public interface QueueFactoryInterface {
|
||||
String CLUSTER_EVENT_NAMED = "clusterEventQueue";
|
||||
String SUBFLOWEXECUTIONEND_NAMED = "subflowExecutionEndQueue";
|
||||
String EXECUTION_RUNNING_NAMED = "executionRunningQueue";
|
||||
String MULTIPLE_CONDITION_EVENT_NAMED = "multipleConditionEventQueue";
|
||||
|
||||
QueueInterface<Execution> execution();
|
||||
|
||||
@@ -59,4 +60,6 @@ public interface QueueFactoryInterface {
|
||||
QueueInterface<SubflowExecutionEnd> subflowExecutionEnd();
|
||||
|
||||
QueueInterface<ExecutionRunning> executionRunning();
|
||||
|
||||
QueueInterface<MultipleConditionEvent> multipleConditionEvent();
|
||||
}
|
||||
|
||||
@@ -25,8 +25,6 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
|
||||
Boolean isTaskRunEnabled();
|
||||
|
||||
default Optional<Execution> findById(String tenantId, String id) {
|
||||
return findById(tenantId, id, false);
|
||||
}
|
||||
@@ -96,24 +94,19 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
|
||||
Flux<Execution> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
ArrayListTotal<TaskRun> findTaskRun(
|
||||
Pageable pageable,
|
||||
@Nullable String tenantId,
|
||||
List<QueryFilter> filters
|
||||
);
|
||||
|
||||
Execution delete(Execution execution);
|
||||
|
||||
Integer purge(Execution execution);
|
||||
|
||||
Integer purge(List<Execution> executions);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatisticsForAllTenants(
|
||||
@Nullable String query,
|
||||
@Nullable String namespace,
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
boolean isTaskRun
|
||||
@Nullable DateUtils.GroupType groupBy
|
||||
);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatistics(
|
||||
@@ -125,8 +118,7 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
List<State.Type> state,
|
||||
boolean isTaskRun
|
||||
List<State.Type> state
|
||||
);
|
||||
|
||||
@Getter
|
||||
|
||||
@@ -4,6 +4,7 @@ import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.plugin.core.dashboard.data.Flows;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -11,7 +12,7 @@ import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface FlowRepositoryInterface {
|
||||
public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fields> {
|
||||
|
||||
Optional<Flow> findById(String tenantId, String namespace, String id, Optional<Integer> revision, Boolean allowDeleted);
|
||||
|
||||
@@ -162,4 +163,6 @@ public interface FlowRepositoryInterface {
|
||||
FlowWithSource update(GenericFlow flow, FlowInterface previous) throws ConstraintViolationException;
|
||||
|
||||
FlowWithSource delete(FlowInterface flow);
|
||||
|
||||
Boolean existAnyNoAcl(String tenantId);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,9 @@ public class LocalFlowRepositoryLoader {
|
||||
}
|
||||
|
||||
public void load(String tenantId, File basePath) throws IOException {
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
|
||||
.stream()
|
||||
.filter(flow -> tenantId.equals(flow.getTenantId()))
|
||||
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
|
||||
|
||||
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {
|
||||
|
||||
@@ -90,6 +90,8 @@ public interface LogRepositoryInterface extends SaveRepositoryInterface<LogEntry
|
||||
|
||||
Integer purge(Execution execution);
|
||||
|
||||
Integer purge(List<Execution> executions);
|
||||
|
||||
void deleteByQuery(String tenantId, String executionId, String taskId, String taskRunId, Level minLevel, Integer attempt);
|
||||
|
||||
void deleteByQuery(String tenantId, String namespace, String flowId, String triggerId);
|
||||
|
||||
@@ -29,6 +29,8 @@ public interface MetricRepositoryInterface extends SaveRepositoryInterface<Metri
|
||||
|
||||
Integer purge(Execution execution);
|
||||
|
||||
Integer purge(List<Execution> executions);
|
||||
|
||||
Flux<MetricEntry> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
default Function<String, String> sortMapping() throws IllegalArgumentException {
|
||||
|
||||
@@ -5,10 +5,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.exceptions.InternalException;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
@@ -29,6 +26,7 @@ import org.apache.commons.lang3.stream.Streams;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.trace.Tracer.throwCallable;
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
@@ -153,14 +151,22 @@ public final class ExecutableUtils {
|
||||
currentFlow.getNamespace(),
|
||||
currentFlow.getId()
|
||||
)
|
||||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
|
||||
.orElseThrow(() -> {
|
||||
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
|
||||
runContext.logger().error(msg);
|
||||
return new IllegalStateException(msg);
|
||||
});
|
||||
|
||||
if (flow.isDisabled()) {
|
||||
throw new IllegalStateException("Cannot execute a flow which is disabled");
|
||||
String msg = "Cannot execute a flow which is disabled";
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
}
|
||||
|
||||
if (flow instanceof FlowWithException fwe) {
|
||||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
|
||||
String msg = "Cannot execute an invalid flow: " + fwe.getException();
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
}
|
||||
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
@@ -201,7 +207,20 @@ public final class ExecutableUtils {
|
||||
.build()
|
||||
)
|
||||
.withScheduleDate(scheduleOnDate);
|
||||
|
||||
if(execution.getInputs().size()<inputs.size()) {
|
||||
Map<String,Object>resolvedInputs=execution.getInputs();
|
||||
for (var inputKey : inputs.keySet()) {
|
||||
if (!resolvedInputs.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
|
||||
inputKey,
|
||||
currentExecution.getId(),
|
||||
currentTask.subflowId().namespace(),
|
||||
currentTask.subflowId().flowId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// inject the traceparent into the new execution
|
||||
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));
|
||||
|
||||
|
||||
@@ -32,5 +32,7 @@ public class ExecutionRunning implements HasUID {
|
||||
return IdUtils.fromPartsAndSeparator('|', this.tenantId, this.namespace, this.flowId, this.execution.getId());
|
||||
}
|
||||
|
||||
public enum ConcurrencyState { CREATED, RUNNING, QUEUED, CANCELLED, FAILED }
|
||||
// Note: the KILLED state is only used in the Kafka implementation to difference between purging terminated running execution (null)
|
||||
// and purging killed execution which need special treatment
|
||||
public enum ConcurrencyState { CREATED, RUNNING, QUEUED, CANCELLED, FAILED, KILLED }
|
||||
}
|
||||
|
||||
@@ -49,15 +49,7 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -231,6 +223,19 @@ public class FlowInputOutput {
|
||||
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
|
||||
})
|
||||
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||
if (resolved.size() < data.size()) {
|
||||
RunContext runContext = runContextFactory.of(flow, execution);
|
||||
for (var inputKey : data.keySet()) {
|
||||
if (!resolved.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
|
||||
inputKey,
|
||||
flow.getNamespace(),
|
||||
flow.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return MapUtils.flattenToNestedMap(resolved);
|
||||
}
|
||||
|
||||
@@ -313,15 +318,15 @@ public class FlowInputOutput {
|
||||
});
|
||||
resolvable.setInput(input);
|
||||
|
||||
|
||||
|
||||
Object value = resolvable.get().value();
|
||||
|
||||
|
||||
// resolve default if needed
|
||||
if (value == null && input.getDefaults() != null) {
|
||||
value = resolveDefaultValue(input, runContext);
|
||||
resolvable.isDefault(true);
|
||||
}
|
||||
|
||||
|
||||
// validate and parse input value
|
||||
if (value == null) {
|
||||
if (input.getRequired()) {
|
||||
@@ -350,7 +355,7 @@ public class FlowInputOutput {
|
||||
|
||||
return resolvable.get();
|
||||
}
|
||||
|
||||
|
||||
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
|
||||
return switch (input.getType()) {
|
||||
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
|
||||
@@ -367,7 +372,7 @@ public class FlowInputOutput {
|
||||
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
|
||||
@@ -376,7 +381,7 @@ public class FlowInputOutput {
|
||||
private static <T> Object resolveDefaultPropertyAsList(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.asList((Property<List<T>>) input.getDefaults(), renderer, clazz);
|
||||
}
|
||||
|
||||
|
||||
private RunContext buildRunContextForExecutionAndInputs(final FlowInterface flow, final Execution execution, Map<String, InputAndValue> dependencies) {
|
||||
Map<String, Object> flattenInputs = MapUtils.flattenToNestedMap(dependencies.entrySet()
|
||||
.stream()
|
||||
@@ -453,7 +458,7 @@ public class FlowInputOutput {
|
||||
if (data.getType() == null) {
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
|
||||
}
|
||||
|
||||
|
||||
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
|
||||
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(
|
||||
@@ -530,17 +535,17 @@ public class FlowInputOutput {
|
||||
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
if (outputs == null) return Map.of();
|
||||
|
||||
|
||||
// render required outputs
|
||||
Map<String, Object> outputsById = outputs
|
||||
.stream()
|
||||
.filter(output -> output.getRequired() == null || output.getRequired())
|
||||
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
|
||||
outputsById = runContext.render(outputsById);
|
||||
|
||||
|
||||
// render optional outputs one by one to catch, log, and skip any error.
|
||||
for (io.kestra.core.models.flows.Output output : outputs) {
|
||||
if (Boolean.FALSE.equals(output.getRequired())) {
|
||||
@@ -583,9 +588,9 @@ public class FlowInputOutput {
|
||||
}
|
||||
|
||||
public void isDefault(boolean isDefault) {
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
}
|
||||
|
||||
|
||||
public void setInput(final Input<?> input) {
|
||||
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
|
||||
}
|
||||
|
||||
@@ -500,7 +500,7 @@ public class FlowableUtils {
|
||||
|
||||
ArrayList<ResolvedTask> result = new ArrayList<>();
|
||||
|
||||
int index = 0;
|
||||
int iteration = 0;
|
||||
for (Object current : distinctValue) {
|
||||
try {
|
||||
String resolvedValue = current instanceof String stringValue ? stringValue : MAPPER.writeValueAsString(current);
|
||||
@@ -508,7 +508,7 @@ public class FlowableUtils {
|
||||
result.add(ResolvedTask.builder()
|
||||
.task(task)
|
||||
.value(resolvedValue)
|
||||
.iteration(index++)
|
||||
.iteration(iteration)
|
||||
.parentId(parentTaskRun.getId())
|
||||
.build()
|
||||
);
|
||||
@@ -516,6 +516,7 @@ public class FlowableUtils {
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalVariableEvaluationException(e);
|
||||
}
|
||||
iteration++;
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
public record MultipleConditionEvent(Flow flow, Execution execution) implements HasUID {
|
||||
@Override
|
||||
public String uid() {
|
||||
return IdUtils.fromParts(flow.uidWithoutRevision(), execution.getId());
|
||||
}
|
||||
}
|
||||
@@ -168,6 +168,7 @@ public class Extension extends AbstractExtension {
|
||||
functions.put("randomPort", new RandomPortFunction());
|
||||
functions.put("fileExists", fileExistsFunction);
|
||||
functions.put("isFileEmpty", isFileEmptyFunction);
|
||||
functions.put("nanoId", new NanoIDFunction());
|
||||
functions.put("tasksWithState", new TasksWithStateFunction());
|
||||
functions.put(HttpFunction.NAME, httpFunction);
|
||||
return functions;
|
||||
|
||||
@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
|
||||
ZoneId zoneId = zoneId(timeZone);
|
||||
ZonedDateTime date = convert(input, zoneId, existingFormat);
|
||||
|
||||
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import io.kestra.core.http.HttpRequest;
|
||||
import io.kestra.core.http.HttpResponse;
|
||||
import io.kestra.core.http.client.HttpClient;
|
||||
import io.kestra.core.http.client.HttpClientException;
|
||||
import io.kestra.core.http.client.HttpClientRequestException;
|
||||
import io.kestra.core.http.client.HttpClientResponseException;
|
||||
import io.kestra.core.http.client.configurations.HttpConfiguration;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
@@ -101,8 +103,15 @@ public class HttpFunction<T> implements Function {
|
||||
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
|
||||
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
|
||||
return response.getBody();
|
||||
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
|
||||
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
|
||||
} catch (HttpClientResponseException e) {
|
||||
if (e.getResponse() != null) {
|
||||
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
|
||||
throw new PebbleException(e, msg , lineNumber, self.getName());
|
||||
} else {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package io.kestra.core.runners.pebble.functions;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Function;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NanoIDFunction implements Function {
|
||||
|
||||
private static final int DEFAULT_LENGTH = 21;
|
||||
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
private static final String LENGTH = "length";
|
||||
private static final String ALPHABET = "alphabet";
|
||||
|
||||
private static final int MAX_LENGTH = 1000;
|
||||
|
||||
@Override
|
||||
public Object execute(
|
||||
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
|
||||
int length = DEFAULT_LENGTH;
|
||||
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
|
||||
length = parseLength(args, self, lineNumber);
|
||||
}
|
||||
char[] alphabet = DEFAULT_ALPHABET;
|
||||
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
|
||||
alphabet = ((String) args.get(ALPHABET)).toCharArray();
|
||||
}
|
||||
return createNanoID(length, alphabet);
|
||||
}
|
||||
|
||||
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
|
||||
var value = (Long) args.get(LENGTH);
|
||||
if(value > MAX_LENGTH) {
|
||||
throw new PebbleException(
|
||||
null,
|
||||
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
|
||||
lineNumber,
|
||||
self.getName());
|
||||
}
|
||||
return Math.toIntExact(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
return List.of(LENGTH,ALPHABET);
|
||||
}
|
||||
|
||||
String createNanoID(int length, char[] alphabet){
|
||||
final char[] data = new char[length];
|
||||
final byte[] bytes = new byte[length];
|
||||
final int mask = alphabet.length-1;
|
||||
secureRandom.nextBytes(bytes);
|
||||
for (int i = 0; i < length; ++i) {
|
||||
data[i] = alphabet[bytes[i] & mask];
|
||||
}
|
||||
return String.valueOf(data);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -180,23 +180,13 @@ public final class FileSerde {
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,8 +56,7 @@ import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
import static io.kestra.core.utils.Rethrow.throwPredicate;
|
||||
import static io.kestra.core.utils.Rethrow.*;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
@@ -431,7 +430,8 @@ public class ExecutionService {
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable List<State.Type> state
|
||||
@Nullable List<State.Type> state,
|
||||
int batchSize
|
||||
) throws IOException {
|
||||
PurgeResult purgeResult = this.executionRepository
|
||||
.find(
|
||||
@@ -448,24 +448,27 @@ public class ExecutionService {
|
||||
null,
|
||||
true
|
||||
)
|
||||
.map(throwFunction(execution -> {
|
||||
.buffer(batchSize)
|
||||
.map(throwFunction(executions -> {
|
||||
PurgeResult.PurgeResultBuilder<?, ?> builder = PurgeResult.builder();
|
||||
|
||||
if (purgeExecution) {
|
||||
builder.executionsCount(this.executionRepository.purge(execution));
|
||||
builder.executionsCount(this.executionRepository.purge(executions));
|
||||
}
|
||||
|
||||
if (purgeLog) {
|
||||
builder.logsCount(this.logRepository.purge(execution));
|
||||
builder.logsCount(this.logRepository.purge(executions));
|
||||
}
|
||||
|
||||
if (purgeMetric) {
|
||||
builder.metricsCount(this.metricRepository.purge(execution));
|
||||
builder.metricsCount(this.metricRepository.purge(executions));
|
||||
}
|
||||
|
||||
if (purgeStorage) {
|
||||
URI uri = StorageContext.forExecution(execution).getExecutionStorageURI(StorageContext.KESTRA_SCHEME);
|
||||
builder.storagesCount(storageInterface.deleteByPrefix(execution.getTenantId(), execution.getNamespace(), uri).size());
|
||||
executions.forEach(throwConsumer(execution -> {
|
||||
URI uri = StorageContext.forExecution(execution).getExecutionStorageURI(StorageContext.KESTRA_SCHEME);
|
||||
builder.storagesCount(storageInterface.deleteByPrefix(execution.getTenantId(), execution.getNamespace(), uri).size());
|
||||
}));
|
||||
}
|
||||
|
||||
return (PurgeResult) builder.build();
|
||||
@@ -716,7 +719,8 @@ public class ExecutionService {
|
||||
newExecution = execution.withState(killingOrAfterKillState);
|
||||
}
|
||||
|
||||
eventPublisher.publishEvent(new CrudEvent<>(newExecution, execution, CrudEventType.UPDATE));
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
// immediately without publishing a CrudEvent like it's done on pause/resume method.
|
||||
return newExecution;
|
||||
}
|
||||
public Execution kill(Execution execution, FlowInterface flow) {
|
||||
|
||||
@@ -1,36 +1,45 @@
|
||||
package io.kestra.executor;
|
||||
package io.kestra.core.services;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class SkipExecutionService {
|
||||
private volatile List<String> skipExecutions = Collections.emptyList();
|
||||
private volatile List<FlowId> skipFlows = Collections.emptyList();
|
||||
private volatile List<NamespaceId> skipNamespaces = Collections.emptyList();
|
||||
private volatile List<String> skipTenants = Collections.emptyList();
|
||||
private volatile List<String> skipIndexerRecords = Collections.emptyList();
|
||||
|
||||
public synchronized void setSkipExecutions(List<String> skipExecutions) {
|
||||
this.skipExecutions = skipExecutions == null ? Collections.emptyList() : skipExecutions;
|
||||
}
|
||||
|
||||
public synchronized void setSkipFlows(List<String> skipFlows) {
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).toList();
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).filter(Objects::nonNull).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipNamespaces(List<String> skipNamespaces) {
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).toList();
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).filter(Objects::nonNull).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipTenants(List<String> skipTenants) {
|
||||
this.skipTenants = skipTenants == null ? Collections.emptyList() : skipTenants;
|
||||
}
|
||||
|
||||
public synchronized void setSkipIndexerRecords(List<String> skipIndexerRecords) {
|
||||
this.skipIndexerRecords = skipIndexerRecords == null ? Collections.emptyList() : skipIndexerRecords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Warning: this method didn't check the flow, so it must be used only when neither of the others can be used.
|
||||
*/
|
||||
@@ -46,6 +55,14 @@ public class SkipExecutionService {
|
||||
return skipExecution(taskRun.getTenantId(), taskRun.getNamespace(), taskRun.getFlowId(), taskRun.getExecutionId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip an indexer records based on its key.
|
||||
* @param key the record key as computed by <code>QueueService.key(record)</code>, can be null
|
||||
*/
|
||||
public boolean skipIndexerRecord(@Nullable String key) {
|
||||
return key != null && skipIndexerRecords.contains(key);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
boolean skipExecution(String tenant, String namespace, String flow, String executionId) {
|
||||
return (tenant != null && skipTenants.contains(tenant)) ||
|
||||
@@ -59,19 +76,31 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
record FlowId(String tenant, String namespace, String flow) {
|
||||
static FlowId from(String flowId) {
|
||||
static @Nullable FlowId from(String flowId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(flowId);
|
||||
if (parts.length == 3) {
|
||||
return new FlowId(parts[0], parts[1], parts[2]);
|
||||
} else if (parts.length == 2) {
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid flow skip with values: '{}'", flowId);
|
||||
}
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
record NamespaceId(String tenant, String namespace) {
|
||||
static NamespaceId from(String namespaceId) {
|
||||
static @Nullable NamespaceId from(String namespaceId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(namespaceId);
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
|
||||
if (parts.length == 2) {
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid namespace skip with values:'{}'", namespaceId);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -32,48 +32,84 @@ public class Version implements Comparable<Version> {
|
||||
* @param version the version.
|
||||
* @return a new {@link Version} instance.
|
||||
*/
|
||||
public static Version of(String version) {
|
||||
public static Version of(final Object version) {
|
||||
|
||||
if (version.startsWith("v")) {
|
||||
version = version.substring(1);
|
||||
if (Objects.isNull(version)) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse null version");
|
||||
}
|
||||
|
||||
String strVersion = version.toString();
|
||||
|
||||
if (strVersion.startsWith("v")) {
|
||||
strVersion = strVersion.substring(1);
|
||||
}
|
||||
|
||||
int qualifier = version.indexOf("-");
|
||||
int qualifier = strVersion.indexOf("-");
|
||||
|
||||
final String[] versions = qualifier > 0 ?
|
||||
version.substring(0, qualifier).split("\\.") :
|
||||
version.split("\\.");
|
||||
strVersion.substring(0, qualifier).split("\\.") :
|
||||
strVersion.split("\\.");
|
||||
try {
|
||||
final int majorVersion = Integer.parseInt(versions[0]);
|
||||
final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0;
|
||||
final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0;
|
||||
final Integer minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : null;
|
||||
final Integer incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : null;
|
||||
|
||||
return new Version(
|
||||
majorVersion,
|
||||
minorVersion,
|
||||
incrementalVersion,
|
||||
qualifier > 0 ? version.substring(qualifier + 1) : null,
|
||||
version
|
||||
qualifier > 0 ? strVersion.substring(qualifier + 1) : null,
|
||||
strVersion
|
||||
);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Static helper method for returning the most recent stable version for a current {@link Version}.
|
||||
* Resolves the most appropriate stable version from a collection, based on a given input version.
|
||||
* <p>
|
||||
* The matching rules are:
|
||||
* <ul>
|
||||
* <li>If {@code from} specifies only a major version (e.g. {@code 1}), return the latest stable version
|
||||
* with the same major (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a major and minor version only (e.g. {@code 1.2}), return the latest
|
||||
* stable version with the same major and minor (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a full version with major, minor, and patch (e.g. {@code 1.2.2}),
|
||||
* then only return it if it is exactly present (and stable) in {@code versions}.
|
||||
* No "upgrade" is performed in this case.</li>
|
||||
* <li>If no suitable version is found, returns {@code null}.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param from the current version.
|
||||
* @param versions the list of version.
|
||||
*
|
||||
* @return the last stable version.
|
||||
* @param from the reference version (may specify only major, or major+minor, or major+minor+patch).
|
||||
* @param versions the collection of candidate versions to resolve against.
|
||||
* @return the best matching stable version, or {@code null} if none match.
|
||||
*/
|
||||
public static Version getStable(final Version from, final Collection<Version> versions) {
|
||||
List<Version> compatibleVersions = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
if (compatibleVersions.isEmpty()) return null;
|
||||
return Version.getLatest(compatibleVersions);
|
||||
// Case 1: "from" is only a major (e.g. 1)
|
||||
if (from.hasOnlyMajor()) {
|
||||
List<Version> sameMajor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion())
|
||||
.toList();
|
||||
return sameMajor.isEmpty() ? null : Version.getLatest(sameMajor);
|
||||
}
|
||||
|
||||
// Case 2: "from" is major+minor only (e.g. 1.2)
|
||||
if (from.hasMajorAndMinorOnly()) {
|
||||
List<Version> sameMinor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion()
|
||||
&& v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
return sameMinor.isEmpty() ? null : Version.getLatest(sameMinor);
|
||||
}
|
||||
|
||||
// Case 3: "from" is full version (major+minor+patch)
|
||||
if (versions.contains(from)) {
|
||||
return from;
|
||||
}
|
||||
|
||||
// No match
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,8 +159,8 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
private final int majorVersion;
|
||||
private final int minorVersion;
|
||||
private final int incrementalVersion;
|
||||
private final Integer minorVersion;
|
||||
private final Integer patchVersion;
|
||||
private final Qualifier qualifier;
|
||||
|
||||
private final String originalVersion;
|
||||
@@ -134,14 +170,14 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
*/
|
||||
public Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
final int patchVersion,
|
||||
final String qualifier) {
|
||||
this(majorVersion, minorVersion, incrementalVersion, qualifier, null);
|
||||
this(majorVersion, minorVersion, patchVersion, qualifier, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -149,25 +185,25 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
* @param originalVersion the original string version.
|
||||
*/
|
||||
private Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
private Version(final Integer majorVersion,
|
||||
final Integer minorVersion,
|
||||
final Integer patchVersion,
|
||||
final String qualifier,
|
||||
final String originalVersion) {
|
||||
this.majorVersion = requirePositive(majorVersion, "major");
|
||||
this.minorVersion = requirePositive(minorVersion, "minor");
|
||||
this.incrementalVersion = requirePositive(incrementalVersion, "incremental");
|
||||
this.patchVersion = requirePositive(patchVersion, "incremental");
|
||||
this.qualifier = qualifier != null ? new Qualifier(qualifier) : null;
|
||||
this.originalVersion = originalVersion;
|
||||
}
|
||||
|
||||
|
||||
private static int requirePositive(int version, final String message) {
|
||||
if (version < 0) {
|
||||
private static Integer requirePositive(Integer version, final String message) {
|
||||
if (version != null && version < 0) {
|
||||
throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message));
|
||||
}
|
||||
return version;
|
||||
@@ -178,11 +214,11 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
public int minorVersion() {
|
||||
return minorVersion;
|
||||
return minorVersion != null ? minorVersion : 0;
|
||||
}
|
||||
|
||||
public int incrementalVersion() {
|
||||
return incrementalVersion;
|
||||
public int patchVersion() {
|
||||
return patchVersion != null ? patchVersion : 0;
|
||||
}
|
||||
|
||||
public Qualifier qualifier() {
|
||||
@@ -197,9 +233,9 @@ public class Version implements Comparable<Version> {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof Version)) return false;
|
||||
Version version = (Version) o;
|
||||
return majorVersion == version.majorVersion &&
|
||||
minorVersion == version.minorVersion &&
|
||||
incrementalVersion == version.incrementalVersion &&
|
||||
return Objects.equals(majorVersion,version.majorVersion) &&
|
||||
Objects.equals(minorVersion, version.minorVersion) &&
|
||||
Objects.equals(patchVersion,version.patchVersion) &&
|
||||
Objects.equals(qualifier, version.qualifier);
|
||||
}
|
||||
|
||||
@@ -208,7 +244,7 @@ public class Version implements Comparable<Version> {
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier);
|
||||
return Objects.hash(majorVersion, minorVersion, patchVersion, qualifier);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -218,7 +254,7 @@ public class Version implements Comparable<Version> {
|
||||
public String toString() {
|
||||
if (originalVersion != null) return originalVersion;
|
||||
|
||||
String version = majorVersion + "." + minorVersion + "." + incrementalVersion;
|
||||
String version = majorVersion + "." + minorVersion + "." + patchVersion;
|
||||
return (qualifier != null) ? version +"-" + qualifier : version;
|
||||
}
|
||||
|
||||
@@ -238,7 +274,7 @@ public class Version implements Comparable<Version> {
|
||||
return compareMinor;
|
||||
}
|
||||
|
||||
int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion);
|
||||
int compareIncremental = Integer.compare(that.patchVersion, this.patchVersion);
|
||||
if (compareIncremental != 0) {
|
||||
return compareIncremental;
|
||||
}
|
||||
@@ -253,6 +289,21 @@ public class Version implements Comparable<Version> {
|
||||
|
||||
return this.qualifier.compareTo(that.qualifier);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true if only major is specified (e.g. "1")
|
||||
*/
|
||||
private boolean hasOnlyMajor() {
|
||||
return minorVersion == null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if major+minor are specified, but no patch (e.g. "1.2")
|
||||
*/
|
||||
private boolean hasMajorAndMinorOnly() {
|
||||
return minorVersion != null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether this version is before the given one.
|
||||
|
||||
@@ -46,16 +46,19 @@ public class VersionProvider {
|
||||
this.date = loadTime(gitProperties);
|
||||
this.version = loadVersion(buildProperties, gitProperties);
|
||||
|
||||
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
|
||||
if (settingRepository.isPresent()) {
|
||||
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
|
||||
settingRepository.get().save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(this.version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
// check the version in the settings and update if needed, we didn't use it would allow us to detect incompatible update later if needed
|
||||
settingRepository.ifPresent(
|
||||
settingRepositoryInterface -> persistVersion(settingRepositoryInterface, version));
|
||||
}
|
||||
|
||||
private static synchronized void persistVersion(SettingRepositoryInterface settingRepositoryInterface, String version) {
|
||||
Optional<Setting> versionSetting = settingRepositoryInterface.findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(version)) {
|
||||
settingRepositoryInterface.save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ public class FlowValidator implements ConstraintValidator<FlowValidation, Flow>
|
||||
|
||||
List<String> violations = new ArrayList<>();
|
||||
|
||||
if (RESERVED_FLOW_IDS.contains(value.getId())) {
|
||||
if (value.getId() != null && RESERVED_FLOW_IDS.contains(value.getId())) {
|
||||
violations.add("Flow id is a reserved keyword: " + value.getId() + ". List of reserved keywords: " + String.join(", ", RESERVED_FLOW_IDS));
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilter;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display Flow data in a dashboard chart."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display a chart with a list of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: list_flows
|
||||
type: io.kestra.plugin.core.dashboard.chart.Table
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.Flows
|
||||
columns:
|
||||
namespace:
|
||||
field: NAMESPACE
|
||||
id:
|
||||
field: ID
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("Flows")
|
||||
public class Flows<C extends ColumnDescriptor<Flows.Fields>> extends DataFilter<Flows.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilterKPI;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display a chart with Flows KPI.",
|
||||
description = "Change."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display count of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: kpi
|
||||
type: io.kestra.plugin.core.dashboard.chart.KPI
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.FlowsKPI
|
||||
columns:
|
||||
field: ID
|
||||
agg: COUNT
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("FlowsKPI")
|
||||
public class FlowsKPI<C extends ColumnDescriptor<FlowsKPI.Fields>> extends DataFilterKPI<FlowsKPI.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.dashboards.filters.AbstractFilter;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public interface IFlows extends IData<IFlows.Fields> {
|
||||
|
||||
default List<AbstractFilter<IFlows.Fields>> whereWithGlobalFilters(List<QueryFilter> filters, ZonedDateTime startDate, ZonedDateTime endDate, List<AbstractFilter<IFlows.Fields>> where) {
|
||||
List<AbstractFilter<IFlows.Fields>> updatedWhere = where != null ? new ArrayList<>(where) : new ArrayList<>();
|
||||
|
||||
if (ListUtils.isEmpty(filters)) {
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
List<QueryFilter> namespaceFilters = filters.stream().filter(f -> f.field().equals(QueryFilter.Field.NAMESPACE)).toList();
|
||||
if (!namespaceFilters.isEmpty()) {
|
||||
updatedWhere.removeIf(filter -> filter.getField().equals(IFlows.Fields.NAMESPACE));
|
||||
namespaceFilters.forEach(f -> {
|
||||
updatedWhere.add(f.toDashboardFilterBuilder(IFlows.Fields.NAMESPACE, f.value()));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
enum Fields {
|
||||
ID,
|
||||
NAMESPACE,
|
||||
REVISION
|
||||
}
|
||||
}
|
||||
@@ -127,9 +127,24 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
}
|
||||
|
||||
// check for system labels: none can be passed at runtime
|
||||
Optional<Map.Entry<String, String>> first = labelsAsMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX)).findFirst();
|
||||
if (first.isPresent()) {
|
||||
throw new IllegalArgumentException("System labels can only be set by Kestra itself, offending label: " + first.get().getKey() + "=" + first.get().getValue());
|
||||
Optional<Map.Entry<String, String>> systemLabel = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX))
|
||||
.findFirst();
|
||||
if (systemLabel.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"System labels can only be set by Kestra itself, offending label: " +
|
||||
systemLabel.get().getKey() + "=" + systemLabel.get().getValue()
|
||||
);
|
||||
}
|
||||
|
||||
// check for empty label values
|
||||
Optional<Map.Entry<String, String>> emptyValue = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().isEmpty())
|
||||
.findFirst();
|
||||
if (emptyValue.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label values cannot be empty, offending label: " + emptyValue.get().getKey()
|
||||
);
|
||||
}
|
||||
|
||||
Map<String, String> newLabels = ListUtils.emptyOnNull(execution.getLabels()).stream()
|
||||
@@ -140,6 +155,7 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
newLabels.putAll(labelsAsMap);
|
||||
|
||||
return execution.withLabels(newLabels.entrySet().stream()
|
||||
.filter(Label.getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(
|
||||
entry.getKey(),
|
||||
entry.getValue()
|
||||
|
||||
@@ -102,6 +102,14 @@ public class PurgeExecutions extends Task implements RunnableTask<PurgeExecution
|
||||
@Builder.Default
|
||||
private Property<Boolean> purgeStorage = Property.ofValue(true);
|
||||
|
||||
@Schema(
|
||||
title = "The size of the bulk delete",
|
||||
description = "For performance, deletion is made by batch of by default 100 executions/logs/metrics."
|
||||
)
|
||||
@Builder.Default
|
||||
@NotNull
|
||||
private Property<Integer> batchSize = Property.ofValue(100);
|
||||
|
||||
@Override
|
||||
public PurgeExecutions.Output run(RunContext runContext) throws Exception {
|
||||
ExecutionService executionService = ((DefaultRunContext)runContext).getApplicationContext().getBean(ExecutionService.class);
|
||||
@@ -124,9 +132,10 @@ public class PurgeExecutions extends Task implements RunnableTask<PurgeExecution
|
||||
flowInfo.tenantId(),
|
||||
renderedNamespace,
|
||||
runContext.render(flowId).as(String.class).orElse(null),
|
||||
startDate != null ? ZonedDateTime.parse(runContext.render(startDate).as(String.class).orElseThrow()) : null,
|
||||
runContext.render(startDate).as(String.class).map(ZonedDateTime::parse).orElse(null),
|
||||
ZonedDateTime.parse(runContext.render(endDate).as(String.class).orElseThrow()),
|
||||
this.states == null ? null : runContext.render(this.states).asList(State.Type.class)
|
||||
this.states == null ? null : runContext.render(this.states).asList(State.Type.class),
|
||||
runContext.render(this.batchSize).as(Integer.class).orElseThrow()
|
||||
);
|
||||
|
||||
return Output.builder()
|
||||
|
||||
@@ -478,7 +478,7 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
try (InputStream is = runContext.storage().getFile(splitsURI)){
|
||||
String fileContent = new String(is.readAllBytes());
|
||||
List<URI> splits = fileContent.lines().map(line -> URI.create(line)).toList();
|
||||
AtomicInteger currentIteration = new AtomicInteger(1);
|
||||
AtomicInteger currentIteration = new AtomicInteger(0);
|
||||
|
||||
return splits
|
||||
.stream()
|
||||
|
||||
@@ -21,10 +21,13 @@ import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class DocumentationGeneratorTest {
|
||||
@Inject
|
||||
JsonSchemaGenerator jsonSchemaGenerator;
|
||||
|
||||
@@ -37,6 +37,7 @@ import lombok.Value;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
@@ -67,6 +68,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@Testcontainers
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
class HttpClientTest {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
@@ -1,19 +1,32 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.validations.ModelValidator;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class LabelTest {
|
||||
@Inject
|
||||
private ModelValidator modelValidator;
|
||||
|
||||
@Test
|
||||
void shouldGetNestedMapGivenDistinctLabels() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label(Label.USERNAME, "test"),
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
new Label(Label.CORRELATION_ID, "id"),
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null)
|
||||
)
|
||||
);
|
||||
|
||||
assertThat(result).isEqualTo(
|
||||
@@ -34,6 +47,18 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toNestedMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGetMapGivenDistinctLabels() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
@@ -59,6 +84,18 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDuplicateLabelsWithKeyOrderKept() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
@@ -73,4 +110,28 @@ class LabelTest {
|
||||
new Label(Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deduplicateShouldIgnoreEmptyAndNull() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldValidateEmpty() {
|
||||
Optional<ConstraintViolationException> validLabelResult = modelValidator.isValid(new Label("foo", "bar"));
|
||||
assertThat(validLabelResult.isPresent()).isFalse();
|
||||
|
||||
Optional<ConstraintViolationException> emptyValueLabelResult = modelValidator.isValid(new Label("foo", ""));
|
||||
assertThat(emptyValueLabelResult.isPresent()).isTrue();
|
||||
|
||||
Optional<ConstraintViolationException> emptyKeyLabelResult = modelValidator.isValid(new Label("", "bar"));
|
||||
assertThat(emptyKeyLabelResult.isPresent()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -13,19 +13,19 @@ import java.util.Map;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ExecutionTest {
|
||||
private static final TaskRun.TaskRunBuilder TASK_RUN = TaskRun.builder()
|
||||
.id("test");
|
||||
|
||||
@Test
|
||||
void hasTaskRunJoinableTrue() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -36,13 +36,15 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableSameState() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State())
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State())
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -51,7 +53,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableFailedExecutionFromExecutor() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -59,7 +62,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -68,7 +72,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartFailed() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -77,7 +82,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -88,7 +94,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartSuccess() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -97,7 +104,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -109,7 +117,8 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableAfterRestart() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -118,7 +127,8 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package io.kestra.core.models.executions;
|
||||
|
||||
import io.kestra.core.models.flows.State;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StateDurationTest {
|
||||
|
||||
|
||||
private static final Instant NOW = Instant.now();
|
||||
private static final Instant ONE = NOW.minus(Duration.ofDays(1000));
|
||||
private static final Instant TWO = ONE.plus(Duration.ofHours(11));
|
||||
private static final Instant THREE = TWO.plus(Duration.ofHours(222));
|
||||
|
||||
@Test
|
||||
void justCreated() {
|
||||
var state = State.of(
|
||||
State.Type.CREATED,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, ONE)
|
||||
)
|
||||
);
|
||||
assertThat(state.getDuration()).isCloseTo(Duration.between(ONE, NOW), Duration.ofMinutes(10));
|
||||
}
|
||||
|
||||
@Test
|
||||
void success() {
|
||||
var state = State.of(
|
||||
State.Type.SUCCESS,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, ONE),
|
||||
new State.History(State.Type.RUNNING, TWO),
|
||||
new State.History(State.Type.SUCCESS, THREE)
|
||||
)
|
||||
);
|
||||
assertThat(state.getDuration()).isEqualTo(Duration.between(ONE, THREE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void isRunning() {
|
||||
var state = State.of(
|
||||
State.Type.RUNNING,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, ONE),
|
||||
new State.History(State.Type.RUNNING, TWO)
|
||||
)
|
||||
);
|
||||
assertThat(state.getDuration()).isCloseTo(Duration.between(ONE, NOW), Duration.ofMinutes(10));
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,11 @@ import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.TriggerRepositoryInterface;
|
||||
import io.kestra.core.runners.RunnerUtils;
|
||||
import io.kestra.core.runners.TestRunnerUtils;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
import io.kestra.core.services.GraphService;
|
||||
import io.kestra.core.utils.GraphUtils;
|
||||
@@ -45,7 +44,7 @@ class FlowGraphTest {
|
||||
private TriggerRepositoryInterface triggerRepositoryInterface;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
@Test
|
||||
void simple() throws IllegalVariableEvaluationException, IOException {
|
||||
@@ -261,10 +260,10 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant1")
|
||||
void subflow() throws IllegalVariableEvaluationException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml");
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml", "tenant1");
|
||||
FlowGraph flowGraph = GraphUtils.flowGraph(flow, null);
|
||||
|
||||
assertThat(flowGraph.getNodes().size()).isEqualTo(6);
|
||||
@@ -293,15 +292,15 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant2")
|
||||
void dynamicIdSubflow() throws IllegalVariableEvaluationException, TimeoutException, QueueException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml").toBuilder().revision(1).build();
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml", "tenant2").toBuilder().revision(1).build();
|
||||
|
||||
IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> graphService.flowGraph(flow, Collections.singletonList("root.launch")));
|
||||
assertThat(illegalArgumentException.getMessage()).isEqualTo("Can't expand subflow task 'launch' because namespace and/or flowId contains dynamic values. This can only be viewed on an execution.");
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
Execution execution = runnerUtils.runOne("tenant2", "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
"namespace", f.getNamespace(),
|
||||
"flowId", "switch"
|
||||
));
|
||||
@@ -373,13 +372,17 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path) throws IOException {
|
||||
return parse(path, MAIN_TENANT);
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path, String tenantId) throws IOException {
|
||||
URL resource = TestsUtils.class.getClassLoader().getResource(path);
|
||||
assert resource != null;
|
||||
|
||||
File file = new File(resource.getFile());
|
||||
|
||||
return YamlParser.parse(file, FlowWithSource.class).toBuilder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.source(Files.readString(file.toPath()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.runners.*;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
@@ -84,8 +85,9 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfile() throws IOException {
|
||||
URI uri = createNsFile(false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "namespace")));
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", namespace)));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
String fetchedContent = new String(fetch.readAllBytes());
|
||||
@@ -95,7 +97,8 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfileFromOtherNs() throws IOException {
|
||||
URI uri = createNsFile(true);
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, true);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "other")));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
@@ -139,8 +142,7 @@ class URIFetcherTest {
|
||||
);
|
||||
}
|
||||
|
||||
private URI createNsFile(boolean nsInAuthority) throws IOException {
|
||||
String namespace = "namespace";
|
||||
private URI createNsFile(String namespace, boolean nsInAuthority) throws IOException {
|
||||
String filePath = "file.txt";
|
||||
storage.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storage.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
|
||||
@@ -10,6 +10,7 @@ import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -33,17 +34,15 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorage() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
var command = ScriptService.replaceInternalStorage(runContext, null, false);
|
||||
assertThat(command).isEqualTo("");
|
||||
|
||||
command = ScriptService.replaceInternalStorage(runContext, "my command", false);
|
||||
assertThat(command).isEqualTo("my command");
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
File localFile = null;
|
||||
@@ -70,12 +69,10 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorageUnicode() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file-龍.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file-龍");
|
||||
|
||||
String internalStorageUri = "kestra://some/file-龍.txt";
|
||||
File localFile = null;
|
||||
@@ -95,12 +92,10 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadInputFiles() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
List<File> filesToDelete = new ArrayList<>();
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
@@ -143,13 +138,11 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadOutputFiles() throws IOException {
|
||||
var runContext = runContextFactory.of();
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
Path path = createFile(tenant, "file");
|
||||
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/main"));
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
assertThat(outputFiles, not(anEmptyMap()));
|
||||
assertThat(outputFiles.get("file.txt")).isEqualTo(URI.create("kestra:///file.txt"));
|
||||
|
||||
@@ -232,4 +225,13 @@ class ScriptServiceTest {
|
||||
.build();
|
||||
return runContextFactory.of(flow, task, execution, taskRun);
|
||||
}
|
||||
|
||||
private static Path createFile(String tenant, String fileName) throws IOException {
|
||||
Path path = Path.of("/tmp/unittest/%s/%s.txt".formatted(tenant, fileName));
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createDirectory(Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
Files.createFile(path);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package io.kestra.core.models.triggers.multipleflows;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import io.kestra.plugin.core.condition.ExecutionFlow;
|
||||
@@ -33,8 +34,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void allDefault() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -50,8 +52,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void daily() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -67,8 +70,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void dailyAdvance() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -84,8 +88,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void hourly() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -102,8 +107,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void minutely() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -115,8 +121,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expiration() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -136,8 +143,9 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -146,20 +154,21 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -168,20 +177,21 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline_Expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -190,16 +200,17 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults()).isEmpty();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeWindow() throws Exception {
|
||||
void dailyTimeWindow() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
LocalTime startTime = LocalTime.now().truncatedTo(ChronoUnit.MINUTES);
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -208,15 +219,16 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void slidingWindow() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -225,13 +237,13 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(TimeWindow sla) {
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(String tenantId, TimeWindow sla) {
|
||||
var multipleCondition = MultipleCondition.builder()
|
||||
.id("condition-multiple")
|
||||
.id("condition-multiple-%s".formatted(tenantId))
|
||||
.conditions(ImmutableMap.of(
|
||||
"flow-a", ExecutionFlow.builder()
|
||||
.flowId(Property.ofValue("flow-a"))
|
||||
@@ -248,6 +260,7 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
Flow flow = Flow.builder()
|
||||
.namespace(NAMESPACE)
|
||||
.id("multiple-flow")
|
||||
.tenantId(tenantId)
|
||||
.revision(1)
|
||||
.triggers(Collections.singletonList(io.kestra.plugin.core.trigger.Flow.builder()
|
||||
.id("trigger-flow")
|
||||
|
||||
@@ -13,21 +13,20 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
public abstract class AbstractFeatureUsageReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
FeatureUsageReport featureUsageReport;
|
||||
|
||||
|
||||
@Test
|
||||
public void shouldGetReport() {
|
||||
// When
|
||||
Instant now = Instant.now();
|
||||
FeatureUsageReport.UsageEvent event = featureUsageReport.report(
|
||||
now,
|
||||
now,
|
||||
Reportable.TimeInterval.of(now.minus(Duration.ofDays(1)).atZone(ZoneId.systemDefault()), now.atZone(ZoneId.systemDefault()))
|
||||
);
|
||||
|
||||
|
||||
// Then
|
||||
assertThat(event.getExecutions().getDailyExecutionsCount().size()).isGreaterThan(0);
|
||||
assertThat(event.getExecutions().getDailyTaskRunsCount()).isNull();
|
||||
}
|
||||
}
|
||||
@@ -22,24 +22,24 @@ import java.util.Set;
|
||||
|
||||
@KestraTest
|
||||
public abstract class AbstractServiceUsageReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
ServiceUsageReport serviceUsageReport;
|
||||
|
||||
|
||||
@Inject
|
||||
ServiceInstanceRepositoryInterface serviceInstanceRepository;
|
||||
|
||||
|
||||
@Test
|
||||
public void shouldGetReport() {
|
||||
// Given
|
||||
final LocalDate start = LocalDate.now().withDayOfMonth(1);
|
||||
final LocalDate start = LocalDate.of(2025, 1, 1);
|
||||
final LocalDate end = start.withDayOfMonth(start.getMonth().length(start.isLeapYear()));
|
||||
final ZoneId zoneId = ZoneId.systemDefault();
|
||||
|
||||
|
||||
LocalDate from = start;
|
||||
int days = 0;
|
||||
// generate one month of service instance
|
||||
|
||||
|
||||
while (from.toEpochDay() < end.toEpochDay()) {
|
||||
Instant createAt = from.atStartOfDay(zoneId).toInstant();
|
||||
Instant updatedAt = from.atStartOfDay(zoneId).plus(Duration.ofHours(10)).toInstant();
|
||||
@@ -62,14 +62,14 @@ public abstract class AbstractServiceUsageReportTest {
|
||||
from = from.plusDays(1);
|
||||
days++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// When
|
||||
Instant now = end.plusDays(1).atStartOfDay(zoneId).toInstant();
|
||||
ServiceUsageReport.ServiceUsageEvent event = serviceUsageReport.report(now,
|
||||
Reportable.TimeInterval.of(start.atStartOfDay(zoneId), end.plusDays(1).atStartOfDay(zoneId))
|
||||
);
|
||||
|
||||
|
||||
// Then
|
||||
List<ServiceUsage.DailyServiceStatistics> statistics = event.services().dailyStatistics();
|
||||
Assertions.assertEquals(ServiceType.values().length - 1, statistics.size());
|
||||
|
||||
@@ -18,10 +18,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class SystemInformationReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
private SystemInformationReport systemInformationReport;
|
||||
|
||||
|
||||
@Test
|
||||
void shouldGetReport() {
|
||||
SystemInformationReport.SystemInformationEvent event = systemInformationReport.report(Instant.now());
|
||||
@@ -32,34 +32,34 @@ class SystemInformationReportTest {
|
||||
assertThat(event.host().getHardware().getLogicalProcessorCount()).isNotNull();
|
||||
assertThat(event.host().getJvm().getName()).isNotNull();
|
||||
assertThat(event.host().getOs().getFamily()).isNotNull();
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("memory");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("memory");
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("h2");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("h2");
|
||||
}
|
||||
|
||||
|
||||
@MockBean(SettingRepositoryInterface.class)
|
||||
@Singleton
|
||||
static class TestSettingRepository implements SettingRepositoryInterface {
|
||||
public static Object UUID = null;
|
||||
|
||||
|
||||
@Override
|
||||
public Optional<Setting> findByKey(String key) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Setting> findAll() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting save(Setting setting) throws ConstraintViolationException {
|
||||
if (setting.getKey().equals(Setting.INSTANCE_UUID)) {
|
||||
UUID = setting.getValue();
|
||||
}
|
||||
|
||||
|
||||
return setting;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting delete(Setting setting) {
|
||||
return setting;
|
||||
|
||||
@@ -25,6 +25,7 @@ import io.kestra.core.models.tasks.ResolvedTask;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.NamespaceUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
@@ -48,7 +49,6 @@ import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
@@ -62,17 +62,17 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId) {
|
||||
return builder(state, flowId, NAMESPACE);
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId) {
|
||||
return builder(tenantId, state, flowId, NAMESPACE);
|
||||
}
|
||||
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId, String namespace) {
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId, String namespace) {
|
||||
State finalState = randomDuration(state);
|
||||
|
||||
Execution.ExecutionBuilder execution = Execution.builder()
|
||||
.id(FriendlyId.createFriendlyId())
|
||||
.namespace(namespace)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.flowId(flowId == null ? FLOW : flowId)
|
||||
.flowRevision(1)
|
||||
.state(finalState);
|
||||
@@ -126,11 +126,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
return finalState;
|
||||
}
|
||||
|
||||
protected void inject() {
|
||||
inject(null);
|
||||
protected void inject(String tenantId) {
|
||||
inject(tenantId, null);
|
||||
}
|
||||
|
||||
protected void inject(String executionTriggerId) {
|
||||
protected void inject(String tenantId, String executionTriggerId) {
|
||||
ExecutionTrigger executionTrigger = null;
|
||||
|
||||
if (executionTriggerId != null) {
|
||||
@@ -139,7 +139,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
executionRepository.save(builder(State.Type.RUNNING, null)
|
||||
executionRepository.save(builder(tenantId, State.Type.RUNNING, null)
|
||||
.labels(List.of(
|
||||
new Label("key", "value"),
|
||||
new Label("key2", "value2")
|
||||
@@ -149,6 +149,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
);
|
||||
for (int i = 1; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).trigger(executionTrigger).build());
|
||||
@@ -156,6 +157,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
// add a test execution, this should be ignored in search & statistics
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
State.Type.SUCCESS,
|
||||
null
|
||||
)
|
||||
@@ -167,9 +169,10 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter, int expectedSize){
|
||||
inject("executionTriggerId");
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, "executionTriggerId");
|
||||
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(expectedSize);
|
||||
}
|
||||
@@ -192,7 +195,8 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -208,9 +212,10 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void find() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -219,7 +224,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value( List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -227,7 +232,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(1L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -235,7 +240,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value2"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -244,7 +249,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(Map.of("key", "value", "keyTest", "valueTest"))
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -252,7 +257,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value("second")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -266,7 +271,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(NAMESPACE)
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -274,7 +279,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.STARTS_WITH)
|
||||
.value("io.kestra")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
}
|
||||
|
||||
@@ -282,15 +287,16 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
protected void findTriggerExecutionId() {
|
||||
String executionTriggerId = IdUtils.create();
|
||||
|
||||
inject(executionTriggerId);
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, executionTriggerId);
|
||||
inject(tenant);
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.TRIGGER_EXECUTION_ID)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(executionTriggerId)
|
||||
.build());
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -300,7 +306,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.CHILD)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -311,20 +317,21 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.MAIN)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters );
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters );
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger()).isNull();
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(56L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findWithSort() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), MAIN_TENANT, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), tenant, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -333,100 +340,107 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findTaskRun() {
|
||||
inject();
|
||||
|
||||
ArrayListTotal<TaskRun> taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(74L);
|
||||
assertThat(taskRuns.size()).isEqualTo(10);
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
|
||||
taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(1L);
|
||||
assertThat(taskRuns.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
protected void findById() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(execution1.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByIdTestExecution() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_TEST);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var executionTest = ExecutionFixture.EXECUTION_TEST(tenant);
|
||||
executionRepository.save(executionTest);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(null, ExecutionFixture.EXECUTION_TEST.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, executionTest.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_TEST.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(executionTest.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void purge() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.purge(ExecutionFixture.EXECUTION_1);
|
||||
executionRepository.purge(execution1);
|
||||
|
||||
full = executionRepository.findById(null, ExecutionFixture.EXECUTION_1.getId());
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void delete() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
protected void purgeExecutions() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
var execution2 = ExecutionFixture.EXECUTION_2(tenant);
|
||||
executionRepository.save(execution2);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
var results = executionRepository.purge(List.of(execution1, execution2));
|
||||
assertThat(results).isEqualTo(2);
|
||||
|
||||
assertThat(executionRepository.findById(tenant, execution1.getId())).isEmpty();
|
||||
assertThat(executionRepository.findById(tenant, execution2.getId())).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void delete() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.delete(ExecutionFixture.EXECUTION_1);
|
||||
executionRepository.delete(execution1);
|
||||
|
||||
full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void mappingConflict() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2(tenant));
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1(tenant));
|
||||
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(MAIN_TENANT, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(tenant, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
|
||||
assertThat(page1.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void dailyStatistics() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
@@ -436,15 +450,14 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
@@ -456,131 +469,52 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(21L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(20L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
false);
|
||||
null);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void taskRunsDailyStatistics() {
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getDuration().getAvg().toMillis()).isGreaterThan(0L);
|
||||
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED)).isEqualTo(3L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.RUNNING)).isEqualTo(5L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(55L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@SuppressWarnings("OptionalGetWithoutIsPresent")
|
||||
@Test
|
||||
protected void executionsCount() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 14; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
i < 2 ? "first" : (i < 5 ? "second" : "third")
|
||||
).build());
|
||||
@@ -590,7 +524,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
Thread.sleep(500);
|
||||
|
||||
List<ExecutionCount> result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -609,7 +543,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("missing")).findFirst().get().getCount()).isEqualTo(0L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -626,7 +560,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("third")).findFirst().get().getCount()).isEqualTo(9L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
MAIN_TENANT,
|
||||
tenant,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
@@ -639,14 +573,15 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void update() {
|
||||
Execution execution = ExecutionFixture.EXECUTION_1;
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution execution = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution);
|
||||
|
||||
Label label = new Label("key", "value");
|
||||
Execution updated = execution.toBuilder().labels(List.of(label)).build();
|
||||
executionRepository.update(updated);
|
||||
|
||||
Optional<Execution> validation = executionRepository.findById(MAIN_TENANT, updated.getId());
|
||||
Optional<Execution> validation = executionRepository.findById(tenant, updated.getId());
|
||||
assertThat(validation.isPresent()).isTrue();
|
||||
assertThat(validation.get().getLabels().size()).isEqualTo(1);
|
||||
assertThat(validation.get().getLabels().getFirst()).isEqualTo(label);
|
||||
@@ -654,13 +589,14 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldFindLatestExecutionGivenState() {
|
||||
Execution earliest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(5)));
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution earliest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(5)));
|
||||
|
||||
executionRepository.save(earliest);
|
||||
executionRepository.save(latest);
|
||||
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(MAIN_TENANT, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(tenant, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
assertThat(result.isPresent()).isTrue();
|
||||
assertThat(result.get().getId()).isEqualTo(latest.getId());
|
||||
}
|
||||
@@ -700,11 +636,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(data.get(0).get("date")).isEqualTo(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").format(ZonedDateTime.ofInstant(startDate, ZoneId.systemDefault()).withSecond(0).withNano(0)));
|
||||
}
|
||||
|
||||
private static Execution buildWithCreatedDate(Instant instant) {
|
||||
private static Execution buildWithCreatedDate(String tenant, Instant instant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State(State.Type.CREATED, List.of(new State.History(State.Type.CREATED, instant))))
|
||||
@@ -715,22 +651,24 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void findAllAsync() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<Execution> executions = executionRepository.findAllAsync(MAIN_TENANT).collectList().block();
|
||||
List<Execution> executions = executionRepository.findAllAsync(tenant).collectList().block();
|
||||
assertThat(executions).hasSize(29); // used by the backup so it contains TEST executions
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByLabel() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<QueryFilter> filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.size()).isEqualTo(1L);
|
||||
|
||||
// Filtering by two pairs of labels, since now its a and behavior, it should not return anything
|
||||
@@ -739,19 +677,107 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value", "keyother", "valueother"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
assertThat(executions.size()).isEqualTo(0L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastExecutionsWhenInputsAreNull() {
|
||||
inject();
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(MAIN_TENANT, null);
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(tenant, null);
|
||||
|
||||
assertThat(lastExecutions).isNotEmpty();
|
||||
Set<String> flowIds = lastExecutions.stream().map(Execution::getFlowId).collect(Collectors.toSet());
|
||||
assertThat(flowIds.size()).isEqualTo(lastExecutions.size());
|
||||
}
|
||||
|
||||
private static final Instant NOW = Instant.now();
|
||||
private static final Instant INSTANT_ONE = NOW.minus(Duration.ofDays(1000));
|
||||
private static final Instant INSTANT_TWO = INSTANT_ONE.plus(Duration.ofHours(11));
|
||||
private static final Instant INSTANT_THREE = INSTANT_TWO.plus(Duration.ofHours(222));
|
||||
|
||||
@Test
|
||||
protected void findShouldSortCorrectlyOnDurationAndDates() {
|
||||
// given
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var createdExecution = Execution.builder()
|
||||
.id("createdExecution__"+FriendlyId.createFriendlyId())
|
||||
.namespace(NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.flowId(FLOW)
|
||||
.flowRevision(1)
|
||||
.state(
|
||||
State.of(
|
||||
State.Type.CREATED,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, INSTANT_ONE)
|
||||
)
|
||||
)
|
||||
).build();
|
||||
assertThat(createdExecution.getState().getDuration()).isCloseTo(Duration.ofDays(1000), Duration.ofMinutes(10));
|
||||
executionRepository.save(createdExecution);
|
||||
|
||||
var successExecution = Execution.builder()
|
||||
.id("successExecution__"+FriendlyId.createFriendlyId())
|
||||
.namespace(NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.flowId(FLOW)
|
||||
.flowRevision(1)
|
||||
.state(
|
||||
State.of(
|
||||
State.Type.SUCCESS,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, INSTANT_ONE),
|
||||
new State.History(State.Type.RUNNING, INSTANT_TWO),
|
||||
new State.History(State.Type.SUCCESS, INSTANT_THREE)
|
||||
)
|
||||
)
|
||||
).build();
|
||||
assertThat(successExecution.getState().getDuration()).isCloseTo(Duration.ofHours(233), Duration.ofMinutes(10));
|
||||
executionRepository.save(successExecution);
|
||||
|
||||
var runningExecution = Execution.builder()
|
||||
.id("runningExecution__"+FriendlyId.createFriendlyId())
|
||||
.namespace(NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.flowId(FLOW)
|
||||
.flowRevision(1)
|
||||
.state(
|
||||
State.of(
|
||||
State.Type.RUNNING,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, INSTANT_TWO),
|
||||
new State.History(State.Type.RUNNING, INSTANT_THREE)
|
||||
)
|
||||
)
|
||||
).build();
|
||||
assertThat(runningExecution.getState().getDuration()).isCloseTo(Duration.ofDays(1000).minus(Duration.ofHours(11)), Duration.ofMinutes(10));
|
||||
executionRepository.save(runningExecution);
|
||||
|
||||
// when
|
||||
List<QueryFilter> emptyFilters = null;
|
||||
var sortedByShortestDuration = executionRepository.find(Pageable.from(Sort.of(Sort.Order.asc("state_duration"))), tenant, emptyFilters);
|
||||
// then
|
||||
assertThat(sortedByShortestDuration.stream())
|
||||
.as("assert order when finding by shortest duration")
|
||||
.usingRecursiveFieldByFieldElementComparatorOnFields("id")
|
||||
.containsExactly(
|
||||
successExecution,
|
||||
runningExecution,
|
||||
createdExecution
|
||||
);
|
||||
|
||||
// when
|
||||
var findByMoreRecentStartDate = executionRepository.find(Pageable.from(1,1, Sort.of(Sort.Order.desc("start_date"))), tenant, emptyFilters);
|
||||
// then
|
||||
assertThat(findByMoreRecentStartDate.stream())
|
||||
.as("assert order when finding by last start date")
|
||||
.usingRecursiveFieldByFieldElementComparatorOnFields("id")
|
||||
.containsExactly(
|
||||
runningExecution
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -114,7 +114,8 @@ public abstract class AbstractExecutionServiceTest {
|
||||
flow.getId(),
|
||||
null,
|
||||
ZonedDateTime.now(),
|
||||
null
|
||||
null,
|
||||
100
|
||||
);
|
||||
|
||||
assertThat(purge.getExecutionsCount()).isEqualTo(1);
|
||||
@@ -132,7 +133,8 @@ public abstract class AbstractExecutionServiceTest {
|
||||
flow.getId(),
|
||||
null,
|
||||
ZonedDateTime.now(),
|
||||
null
|
||||
null,
|
||||
100
|
||||
);
|
||||
|
||||
assertThat(purge.getExecutionsCount()).isZero();
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.Helpers;
|
||||
import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.exceptions.InvalidQueryFiltersException;
|
||||
@@ -10,7 +9,6 @@ import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.QueryFilter.Field;
|
||||
import io.kestra.core.models.QueryFilter.Op;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.conditions.ConditionContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.ExecutionTrigger;
|
||||
@@ -20,7 +18,6 @@ import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.models.triggers.PollingTriggerInterface;
|
||||
import io.kestra.core.models.triggers.TriggerContext;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.services.FlowService;
|
||||
import io.kestra.core.utils.Await;
|
||||
@@ -29,22 +26,18 @@ import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.slf4j.event.Level;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
@@ -52,16 +45,14 @@ import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
// If some counts are wrong in this test it means that one of the tests is not properly deleting what it created
|
||||
@KestraTest
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public abstract class AbstractFlowRepositoryTest {
|
||||
public static final String TEST_TENANT_ID = "tenant";
|
||||
public static final String TEST_NAMESPACE = "io.kestra.unittest";
|
||||
public static final String TEST_FLOW_ID = "test";
|
||||
@Inject
|
||||
@@ -70,21 +61,18 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@Inject
|
||||
private LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
TestsUtils.loads(MAIN_TENANT, repositoryLoader);
|
||||
@BeforeAll
|
||||
protected static void init() {
|
||||
FlowListener.reset();
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder() {
|
||||
return builder(IdUtils.create(), TEST_FLOW_ID);
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, IdUtils.create(), TEST_FLOW_ID);
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String flowId, String taskId) {
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId, String flowId, String taskId) {
|
||||
return FlowWithSource.builder()
|
||||
.tenantId(tenantId)
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tasks(Collections.singletonList(Return.builder().id(taskId).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()));
|
||||
@@ -93,16 +81,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -113,16 +101,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_with_source(QueryFilter filter){
|
||||
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -144,7 +132,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> flowRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -153,7 +141,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all_with_source(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -176,17 +164,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
Optional<Flow> full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -195,17 +183,18 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithoutAcl() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -214,15 +203,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithSource() {
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
String source = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, source));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, source));
|
||||
|
||||
try {
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
@@ -237,7 +227,8 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
FlowWithSource flow = builder().revision(12).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).revision(12).build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -249,7 +240,8 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void saveNoRevision() {
|
||||
FlowWithSource flow = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -260,68 +252,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
List<Flow> save = flowRepository.findAll(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSource(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
List<Flow> save = flowRepository.findAllForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSourceForAllTenants() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSourceForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
List<Flow> save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 24);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefix() {
|
||||
List<Flow> save = flowRepository.findByNamespacePrefix(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespaceWithSource() {
|
||||
Flow flow = builder()
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
String flowSource = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, flowSource));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, flowSource));
|
||||
|
||||
try {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(MAIN_TENANT, flow.getNamespace());
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(tenant, flow.getNamespace());
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
assertThat(save.getFirst().getSource()).isEqualTo(FlowService.cleanupSource(flowSource));
|
||||
@@ -330,175 +271,15 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefixWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespacePrefixWithSource(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationPartial() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationGreaterThanExisting() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_prefixMatchingAllNamespaces() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecifiedNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificSubNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificFlowByNamespaceAndLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_noResult_forAnUnknownNamespace() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findSpecialChars() {
|
||||
ArrayListTotal<SearchResult<Flow>> save = flowRepository.findSourceCode(Pageable.unpaged(), "https://api.chucknorris.io", MAIN_TENANT, null);
|
||||
assertThat((long) save.size()).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void delete() {
|
||||
Flow flow = builder().tenantId(MAIN_TENANT).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant).tenantId(tenant).build();
|
||||
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
} catch (Throwable e) {
|
||||
deleteFlow(save);
|
||||
throw e;
|
||||
@@ -506,21 +287,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow delete = flowRepository.delete(save);
|
||||
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(tenant, flow.getNamespace(), flow.getId());
|
||||
assertThat(revisions.getLast().getRevision()).isEqualTo(delete.getRevision());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateConflict() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("a").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -528,12 +310,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest2")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("b").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -551,13 +333,14 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void removeTrigger() throws TimeoutException, QueueException {
|
||||
public void removeTrigger() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -567,12 +350,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
;
|
||||
@@ -583,21 +366,25 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
deleteFlow(flow);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.getEmits().size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void removeTriggerDelete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -607,40 +394,39 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(save);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.getEmits().size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findDistinctNamespace() {
|
||||
List<String> distinctNamespace = flowRepository.findDistinctNamespace(MAIN_TENANT);
|
||||
assertThat((long) distinctNamespace.size()).isEqualTo(9L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnNullRevisionForNonExistingFlow() {
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
assertThat(flowRepository.lastRevision(TestsUtils.randomTenant(this.getClass().getSimpleName()), TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastRevisionOnCreate() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId);
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId);
|
||||
|
||||
// Then
|
||||
assertThat(result).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -649,34 +435,36 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final String flowId = IdUtils.create();
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
|
||||
// When
|
||||
flowRepository.delete(created);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnCreateAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
flowRepository.delete(
|
||||
flowRepository.create(createTestingLogFlow(flowId, "first"))
|
||||
flowRepository.create(createTestingLogFlow(tenant, flowId, "first"))
|
||||
);
|
||||
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "second")));
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "second")));
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -685,22 +473,23 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldReturnNullForLastRevisionAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isNull();
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isNull();
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -709,22 +498,23 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldFindAllRevisionsAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -732,21 +522,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnUpdateGivenNotEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
@@ -755,48 +546,39 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldNotIncrementRevisionOnUpdateGivenEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "first"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "first"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnForGivenQueryWildCardFilters() {
|
||||
List<QueryFilter> filters = List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.QUERY).operation(QueryFilter.Op.EQUALS).value("*").build()
|
||||
);
|
||||
ArrayListTotal<Flow> flows = flowRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(flows.size()).isEqualTo(10);
|
||||
assertThat(flows.getTotal()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByExecution() {
|
||||
Flow flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(1)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenant)
|
||||
.flowId(flow.getId())
|
||||
.flowRevision(flow.getRevision())
|
||||
.state(new State())
|
||||
@@ -821,11 +603,13 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByExecutionNoRevision() {
|
||||
Flow flow = builder()
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
.revision(3)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.tenantId(tenant)
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.flowId(flow.getId())
|
||||
@@ -851,16 +635,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource toDelete = null;
|
||||
try {
|
||||
// Given
|
||||
Flow flow = createTestFlowForNamespace(TEST_NAMESPACE);
|
||||
Flow flow = createTestFlowForNamespace(tenant, TEST_NAMESPACE);
|
||||
toDelete = flowRepository.create(GenericFlow.of(flow));
|
||||
// When
|
||||
int count = flowRepository.count(MAIN_TENANT);
|
||||
int count = flowRepository.count(tenant);
|
||||
|
||||
// Then
|
||||
Assertions.assertTrue(count > 0);
|
||||
assertTrue(count > 0);
|
||||
} finally {
|
||||
Optional.ofNullable(toDelete).ifPresent(flow -> {
|
||||
flowRepository.delete(flow);
|
||||
@@ -868,11 +653,41 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static Flow createTestFlowForNamespace(String namespace) {
|
||||
@Test
|
||||
void should_exist_for_tenant(){
|
||||
String tenantFlowExist = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flowExist = FlowWithSource.builder()
|
||||
.id("flowExist")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenantFlowExist)
|
||||
.deleted(false)
|
||||
.build();
|
||||
flowExist = flowRepository.create(GenericFlow.of(flowExist));
|
||||
|
||||
String tenantFlowDeleted = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flowDeleted = FlowWithSource.builder()
|
||||
.id("flowDeleted")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenantFlowDeleted)
|
||||
.deleted(true)
|
||||
.build();
|
||||
flowDeleted = flowRepository.create(GenericFlow.of(flowDeleted));
|
||||
|
||||
try {
|
||||
assertTrue(flowRepository.existAnyNoAcl(tenantFlowExist));
|
||||
assertFalse(flowRepository.existAnyNoAcl("not_found"));
|
||||
assertFalse(flowRepository.existAnyNoAcl(tenantFlowDeleted));
|
||||
} finally {
|
||||
deleteFlow(flowExist);
|
||||
deleteFlow(flowDeleted);
|
||||
}
|
||||
}
|
||||
|
||||
private static Flow createTestFlowForNamespace(String tenantId, String namespace) {
|
||||
return Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.tasks(List.of(Return.builder()
|
||||
.id(IdUtils.create())
|
||||
.type(Return.class.getName())
|
||||
@@ -891,21 +706,31 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
|
||||
@Getter
|
||||
private static List<CrudEvent<Flow>> emits = new ArrayList<>();
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<AbstractFlow>> {
|
||||
private static List<CrudEvent<AbstractFlow>> emits = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<Flow> event) {
|
||||
emits.add(event);
|
||||
public void onApplicationEvent(CrudEvent<AbstractFlow> event) {
|
||||
//This has to be done because Micronaut may send CrudEvent<Setting> for example, and we don't want them.
|
||||
if ((event.getModel() != null && event.getModel() instanceof AbstractFlow)||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof AbstractFlow)) {
|
||||
emits.add(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
emits = new ArrayList<>();
|
||||
emits = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
public static List<CrudEvent<AbstractFlow>> filterByTenant(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getModel() != null && e.getModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
private static GenericFlow createTestingLogFlow(String id, String logMessage) {
|
||||
private static GenericFlow createTestingLogFlow(String tenantId, String id, String logMessage) {
|
||||
String source = """
|
||||
id: %s
|
||||
namespace: %s
|
||||
@@ -914,7 +739,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: %s
|
||||
""".formatted(id, TEST_NAMESPACE, logMessage);
|
||||
return GenericFlow.fromYaml(TEST_TENANT_ID, source);
|
||||
return GenericFlow.fromYaml(tenantId, source);
|
||||
}
|
||||
|
||||
protected static int COUNTER = 0;
|
||||
|
||||
@@ -4,7 +4,7 @@ import io.kestra.core.models.topologies.FlowNode;
|
||||
import io.kestra.core.models.topologies.FlowRelation;
|
||||
import io.kestra.core.models.topologies.FlowTopology;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.tenant.TenantService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -17,21 +17,21 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
@Inject
|
||||
private FlowTopologyRepositoryInterface flowTopologyRepository;
|
||||
|
||||
protected FlowTopology createSimpleFlowTopology(String flowA, String flowB, String namespace) {
|
||||
protected FlowTopology createSimpleFlowTopology(String tenantId, String flowA, String flowB, String namespace) {
|
||||
return FlowTopology.builder()
|
||||
.relation(FlowRelation.FLOW_TASK)
|
||||
.source(FlowNode.builder()
|
||||
.id(flowA)
|
||||
.namespace(namespace)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowA)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowA)
|
||||
.build()
|
||||
)
|
||||
.destination(FlowNode.builder()
|
||||
.id(flowB)
|
||||
.namespace(namespace)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowB)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowB)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
@@ -39,42 +39,45 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByFlow() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(TenantService.MAIN_TENANT, "io.kestra.tests", "flow-a", false);
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(tenant, "io.kestra.tests", "flow-a", false);
|
||||
|
||||
assertThat(list.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(TenantService.MAIN_TENANT, "io.kestra.tests");
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(tenant, "io.kestra.tests");
|
||||
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology("flow-e", "flow-f", "io.kestra.tests.2")
|
||||
createSimpleFlowTopology(tenant, "flow-e", "flow-f", "io.kestra.tests.2")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(TenantService.MAIN_TENANT);
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(tenant);
|
||||
|
||||
assertThat(list.size()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import io.kestra.core.models.executions.LogEntry;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.kestra.plugin.core.dashboard.data.Logs;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -32,9 +34,7 @@ import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatReflectiveOperationException;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@@ -42,11 +42,11 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@Inject
|
||||
protected LogRepositoryInterface logRepository;
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level) {
|
||||
return logEntry(level, IdUtils.create());
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level) {
|
||||
return logEntry(tenantId, level, IdUtils.create());
|
||||
}
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level, String executionId) {
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level, String executionId) {
|
||||
return LogEntry.builder()
|
||||
.flowId("flowId")
|
||||
.namespace("io.kestra.unittest")
|
||||
@@ -57,7 +57,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
.timestamp(Instant.now())
|
||||
.level(level)
|
||||
.thread("")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.triggerId("triggerId")
|
||||
.message("john doe");
|
||||
}
|
||||
@@ -65,9 +65,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -75,9 +76,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_async(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAsync(MAIN_TENANT, List.of(filter));
|
||||
Flux<LogEntry> find = logRepository.findAsync(tenant, List.of(filter));
|
||||
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(1);
|
||||
@@ -86,11 +88,12 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_delete_with_filter(QueryFilter filter){
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
|
||||
logRepository.deleteByFilters(MAIN_TENANT, List.of(filter));
|
||||
logRepository.deleteByFilters(tenant, List.of(filter));
|
||||
|
||||
assertThat(logRepository.findAllAsync(MAIN_TENANT).collectList().block()).isEmpty();
|
||||
assertThat(logRepository.findAllAsync(tenant).collectList().block()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
@@ -150,7 +153,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
() -> logRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
TestsUtils.randomTenant(this.getClass().getSimpleName()),
|
||||
List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -168,16 +174,17 @@ public abstract class AbstractLogRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
|
||||
LogEntry save = logRepository.save(builder.build());
|
||||
logRepository.save(builder.executionKind(ExecutionKind.TEST).build()); // should only be loaded by execution id
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
@@ -193,7 +200,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
find = logRepository.find(Pageable.UNPAGED, "doe", filters);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
@@ -201,141 +208,146 @@ public abstract class AbstractLogRepositoryTest {
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
List<LogEntry> list = logRepository.findByExecutionId(MAIN_TENANT, save.getExecutionId(), null);
|
||||
List<LogEntry> list = logRepository.findByExecutionId(tenant, save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
list = logRepository.findByExecutionId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(tenant, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
Integer countDeleted = logRepository.purge(Execution.builder().id(save.getExecutionId()).build());
|
||||
assertThat(countDeleted).isEqualTo(2);
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageable() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = "123";
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
builder.executionId(executionId);
|
||||
|
||||
for (int i = 0; i < 80; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
builder = logEntry(Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
builder = logEntry(tenant, Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
LogEntry logEntry2 = logRepository.save(builder.build());
|
||||
for (int i = 0; i < 20; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(50);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(3, 50));
|
||||
find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(3, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionIdAndTaskId(tenant, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(21);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindByExecutionIdTestLogs() {
|
||||
var builder = logEntry(Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var builder = logEntry(tenant, Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
logRepository.save(builder);
|
||||
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(MAIN_TENANT, builder.getExecutionId(), null);
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(tenant, builder.getExecutionId(), null);
|
||||
assertThat(logs).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteByQuery() {
|
||||
LogEntry log1 = logEntry(Level.INFO).build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry log1 = logEntry(tenant, Level.INFO).build();
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, log1.getExecutionId(), null, null, null, null);
|
||||
logRepository.deleteByQuery(tenant, log1.getExecutionId(), null, null, null, null);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null);
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null);
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(MAIN_TENANT, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(tenant, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
logRepository.save(logEntry(Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(Level.ERROR).build());
|
||||
logRepository.save(logEntry(Level.WARN).build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(tenant, Level.ERROR).build());
|
||||
logRepository.save(logEntry(tenant, Level.WARN).build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(MAIN_TENANT);
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(tenant);
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fetchData() throws IOException {
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
|
||||
var results = logRepository.fetchData(MAIN_TENANT,
|
||||
var results = logRepository.fetchData(tenant,
|
||||
Logs.builder()
|
||||
.type(Logs.class.getName())
|
||||
.columns(Map.of(
|
||||
@@ -348,4 +360,16 @@ public abstract class AbstractLogRepositoryTest {
|
||||
|
||||
assertThat(results).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void purge() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution2").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution2").build());
|
||||
|
||||
var result = logRepository.purge(List.of(Execution.builder().id("execution1").build(), Execution.builder().id("execution2").build()));
|
||||
assertThat(result).isEqualTo(4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import com.devskiller.friendly_id.FriendlyId;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.ExecutionKind;
|
||||
import io.kestra.core.models.executions.MetricEntry;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.executions.metrics.Counter;
|
||||
import io.kestra.core.models.executions.metrics.MetricAggregations;
|
||||
import io.kestra.core.models.executions.metrics.Timer;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.event.Level;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
@@ -25,27 +28,28 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
MetricEntry testCounter = MetricEntry.of(taskRun1, counter("test"), ExecutionKind.TEST);
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(testCounter); // should only be retrieved by execution id
|
||||
metricRepository.save(timer);
|
||||
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(null, executionId, Pageable.from(1, 10));
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(tenant, executionId, Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskId(null, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskId(tenant, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(null, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(tenant, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(2);
|
||||
|
||||
MetricAggregations aggregationResults = metricRepository.aggregateByFlowId(
|
||||
null,
|
||||
tenant,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -59,7 +63,7 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
assertThat(aggregationResults.getGroupBy()).isEqualTo("day");
|
||||
|
||||
aggregationResults = metricRepository.aggregateByFlowId(
|
||||
null,
|
||||
tenant,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -76,11 +80,12 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void names() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
|
||||
TaskRun taskRun2 = taskRun(executionId, "task2");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task2");
|
||||
MetricEntry counter2 = MetricEntry.of(taskRun2, counter("counter2"), null);
|
||||
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
@@ -90,9 +95,9 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
metricRepository.save(test); // should only be retrieved by execution id
|
||||
|
||||
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(null, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(null, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(null, "namespace", "flow");
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(tenant, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(tenant, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(tenant, "namespace", "flow");
|
||||
|
||||
assertThat(flowMetricsNames.size()).isEqualTo(2);
|
||||
assertThat(taskMetricsNames.size()).isEqualTo(1);
|
||||
@@ -101,20 +106,33 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(timer);
|
||||
metricRepository.save(test); // should be retrieved as findAllAsync is used for backup
|
||||
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(null).collectList().block();
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(tenant).collectList().block();
|
||||
assertThat(results).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void purge() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution1", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution1", "task"), counter("counter2"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution2", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution2", "task"), counter("counter2"), null));
|
||||
|
||||
var result = metricRepository.purge(List.of(Execution.builder().id("execution1").build(), Execution.builder().id("execution2").build()));
|
||||
assertThat(result).isEqualTo(4);
|
||||
}
|
||||
|
||||
private Counter counter(String metricName) {
|
||||
return Counter.of(metricName, 1);
|
||||
}
|
||||
@@ -123,8 +141,9 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
return Timer.of("counter", Duration.ofSeconds(5));
|
||||
}
|
||||
|
||||
private TaskRun taskRun(String executionId, String taskId) {
|
||||
private TaskRun taskRun(String tenantId, String executionId, String taskId) {
|
||||
return TaskRun.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId("flow")
|
||||
.namespace("namespace")
|
||||
.executionId(executionId)
|
||||
|
||||
@@ -4,6 +4,8 @@ import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.templates.Template;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
@@ -11,7 +13,10 @@ import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -20,6 +25,8 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -28,55 +35,60 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
@Inject
|
||||
protected TemplateRepositoryInterface templateRepository;
|
||||
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
@BeforeAll
|
||||
protected static void init() throws IOException, URISyntaxException {
|
||||
TemplateListener.reset();
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder() {
|
||||
return builder(null);
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, null);
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String namespace) {
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId, String namespace) {
|
||||
return Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace == null ? "kestra.test" : namespace)
|
||||
.tenantId(tenantId)
|
||||
.tasks(Collections.singletonList(Return.builder().id("test").type(Return.class.getName()).format(Property.ofValue("test")).build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
|
||||
Optional<Template> full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
Optional<Template> full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
|
||||
full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
Template template1 = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
Template template2 = Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.tenantId(tenant)
|
||||
.namespace("kestra.test.template").build();
|
||||
|
||||
templateRepository.create(template1);
|
||||
templateRepository.create(template2);
|
||||
|
||||
List<Template> templates = templateRepository.findByNamespace(null, template1.getNamespace());
|
||||
List<Template> templates = templateRepository.findByNamespace(tenant, template1.getNamespace());
|
||||
assertThat(templates.size()).isGreaterThanOrEqualTo(1);
|
||||
templates = templateRepository.findByNamespace(null, template2.getNamespace());
|
||||
templates = templateRepository.findByNamespace(tenant, template2.getNamespace());
|
||||
assertThat(templates.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
Template save = templateRepository.create(template);
|
||||
|
||||
assertThat(save.getId()).isEqualTo(template.getId());
|
||||
@@ -84,41 +96,42 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
long saveCount = templateRepository.findAll(null).size();
|
||||
Template template = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAll(tenant).size();
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAll(null).size();
|
||||
long size = templateRepository.findAll(tenant).size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAll(null).size()).isEqualTo(saveCount);
|
||||
assertThat((long) templateRepository.findAll(tenant).size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAllForAllTenants().size();
|
||||
Template template = builder().build();
|
||||
Template template = builder(tenant).build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAllForAllTenants().size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAllForAllTenants().size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find() {
|
||||
Template template1 = builder().build();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
templateRepository.create(template1);
|
||||
Template template2 = builder().build();
|
||||
Template template2 = builder(tenant).build();
|
||||
templateRepository.create(template2);
|
||||
Template template3 = builder().build();
|
||||
Template template3 = builder(tenant).build();
|
||||
templateRepository.create(template3);
|
||||
|
||||
// with pageable
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, null, "kestra.test");
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, tenant, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
// without pageable
|
||||
save = templateRepository.find(null, null, "kestra.test");
|
||||
save = templateRepository.find(null, tenant, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
templateRepository.delete(template1);
|
||||
@@ -126,31 +139,45 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
templateRepository.delete(template3);
|
||||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractTemplateRepositoryTest.class);
|
||||
|
||||
@Test
|
||||
void delete() {
|
||||
Template template = builder().build();
|
||||
protected void delete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
|
||||
Template save = templateRepository.create(template);
|
||||
templateRepository.delete(save);
|
||||
|
||||
assertThat(templateRepository.findById(null, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
assertThat(templateRepository.findById(tenant, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
|
||||
assertThat(TemplateListener.getEmits().size()).isEqualTo(2);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> {
|
||||
LOG.info("-------------> number of event: {}", TemplateListener.getEmits(tenant).size());
|
||||
return TemplateListener.getEmits(tenant).size() == 2;
|
||||
|
||||
}, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class TemplateListener implements ApplicationEventListener<CrudEvent<Template>> {
|
||||
private static List<CrudEvent<Template>> emits = new ArrayList<>();
|
||||
private static List<CrudEvent<Template>> emits = new CopyOnWriteArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<Template> event) {
|
||||
emits.add(event);
|
||||
//The instanceOf is required because Micronaut may send non Template event via this method
|
||||
if ((event.getModel() != null && event.getModel() instanceof Template) ||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof Template)) {
|
||||
emits.add(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static List<CrudEvent<Template>> getEmits() {
|
||||
return emits;
|
||||
public static List<CrudEvent<Template>> getEmits(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getModel() != null && e.getModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
|
||||
@@ -9,6 +9,7 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -24,7 +25,6 @@ import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@@ -35,8 +35,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Inject
|
||||
protected TriggerRepositoryInterface triggerRepository;
|
||||
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger() {
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger(String tenantId) {
|
||||
return Trigger.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId(IdUtils.create())
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.triggerId(IdUtils.create())
|
||||
@@ -44,9 +45,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
.date(ZonedDateTime.now());
|
||||
}
|
||||
|
||||
protected static Trigger generateDefaultTrigger(){
|
||||
protected static Trigger generateDefaultTrigger(String tenantId){
|
||||
Trigger trigger = Trigger.builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tenantId(tenantId)
|
||||
.triggerId("triggerId")
|
||||
.namespace("trigger.namespace")
|
||||
.flowId("flowId")
|
||||
@@ -59,9 +60,10 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -69,9 +71,10 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_async(QueryFilter filter){
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
|
||||
List<Trigger> entries = triggerRepository.find(MAIN_TENANT, List.of(filter)).collectList().block();
|
||||
List<Trigger> entries = triggerRepository.find(tenant, List.of(filter)).collectList().block();
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -92,7 +95,7 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -110,7 +113,8 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger(tenant);
|
||||
|
||||
Optional<Trigger> findLast = triggerRepository.findLast(builder.build());
|
||||
assertThat(findLast.isPresent()).isFalse();
|
||||
@@ -130,47 +134,47 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
assertThat(findLast.get().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
|
||||
triggerRepository.save(trigger().build());
|
||||
triggerRepository.save(trigger().build());
|
||||
Trigger searchedTrigger = trigger().build();
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
Trigger searchedTrigger = trigger(tenant).build();
|
||||
triggerRepository.save(searchedTrigger);
|
||||
|
||||
List<Trigger> all = triggerRepository.findAllForAllTenants();
|
||||
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
assertThat(all.size()).isGreaterThanOrEqualTo(4);
|
||||
|
||||
all = triggerRepository.findAll(null);
|
||||
all = triggerRepository.findAll(tenant);
|
||||
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
|
||||
String namespacePrefix = "io.kestra.another";
|
||||
String namespace = namespacePrefix + ".ns";
|
||||
Trigger trigger = trigger().namespace(namespace).build();
|
||||
Trigger trigger = trigger(tenant).namespace(namespace).build();
|
||||
triggerRepository.save(trigger);
|
||||
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, null, null);
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(4);
|
||||
assertThat(find.getFirst().getNamespace()).isEqualTo(namespace);
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, searchedTrigger.getFlowId(), null);
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, searchedTrigger.getFlowId(), null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getFlowId()).isEqualTo(searchedTrigger.getFlowId());
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, null, namespacePrefix, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, tenant, namespacePrefix, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
|
||||
// Full text search is on namespace, flowId, triggerId, executionId
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), null, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), tenant, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
}
|
||||
@@ -178,15 +182,17 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(Trigger
|
||||
.builder()
|
||||
.tenantId(tenant)
|
||||
.triggerId(IdUtils.create())
|
||||
.flowId(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.build()
|
||||
);
|
||||
// When
|
||||
int count = triggerRepository.count(null);
|
||||
int count = triggerRepository.count(tenant);
|
||||
// Then
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@@ -1,88 +1,92 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
|
||||
class ExecutionFixture {
|
||||
public static final Execution EXECUTION_1 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
public static Execution EXECUTION_1(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static final Execution EXECUTION_2 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
public static Execution EXECUTION_2(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static final Execution EXECUTION_TEST = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
public static Execution EXECUTION_TEST(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
@@ -13,36 +9,36 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.plugin.core.flow.EachSequentialTest;
|
||||
import io.kestra.plugin.core.flow.FlowCaseTest;
|
||||
import io.kestra.plugin.core.flow.ForEachItemCaseTest;
|
||||
import io.kestra.plugin.core.flow.PauseTest;
|
||||
import io.kestra.plugin.core.flow.LoopUntilCaseTest;
|
||||
import io.kestra.plugin.core.flow.WorkingDirectoryTest;
|
||||
import io.kestra.plugin.core.flow.*;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
//@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
|
||||
// must be per-class to allow calling once init() which took a lot of time
|
||||
public abstract class AbstractRunnerTest {
|
||||
|
||||
public static final String TENANT_1 = "tenant1";
|
||||
public static final String TENANT_2 = "tenant2";
|
||||
@Inject
|
||||
protected RunnerUtils runnerUtils;
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
|
||||
protected QueueInterface<LogEntry> logsQueue;
|
||||
|
||||
@Inject
|
||||
private RestartCaseTest restartCaseTest;
|
||||
protected RestartCaseTest restartCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowTriggerCaseTest flowTriggerCaseTest;
|
||||
@@ -54,13 +50,13 @@ public abstract class AbstractRunnerTest {
|
||||
private PluginDefaultsCaseTest pluginDefaultsCaseTest;
|
||||
|
||||
@Inject
|
||||
private FlowCaseTest flowCaseTest;
|
||||
protected FlowCaseTest flowCaseTest;
|
||||
|
||||
@Inject
|
||||
private WorkingDirectoryTest.Suite workingDirectoryTest;
|
||||
|
||||
@Inject
|
||||
private PauseTest.Suite pauseTest;
|
||||
protected PauseTest.Suite pauseTest;
|
||||
|
||||
@Inject
|
||||
private SkipExecutionCaseTest skipExecutionCaseTest;
|
||||
@@ -72,10 +68,10 @@ public abstract class AbstractRunnerTest {
|
||||
protected LoopUntilCaseTest loopUntilTestCaseTest;
|
||||
|
||||
@Inject
|
||||
private FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
protected FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
|
||||
@Inject
|
||||
private ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
protected ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowInputOutput flowIO;
|
||||
@@ -84,7 +80,7 @@ public abstract class AbstractRunnerTest {
|
||||
private SLATestCase slaTestCase;
|
||||
|
||||
@Inject
|
||||
private ChangeStateTestCase changeStateTestCase;
|
||||
protected ChangeStateTestCase changeStateTestCase;
|
||||
|
||||
@Inject
|
||||
private AfterExecutionTestCase afterExecutionTestCase;
|
||||
@@ -115,7 +111,7 @@ public abstract class AbstractRunnerTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(8);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/parallel-nested.yaml")
|
||||
void parallelNested(Execution execution) {
|
||||
assertThat(execution.getTaskRunList()).hasSize(11);
|
||||
@@ -157,27 +153,27 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedThenSuccess();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
void replay() throws Exception {
|
||||
restartCaseTest.replay();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/failed-first.yaml"})
|
||||
void restartMultiple() throws Exception {
|
||||
restartCaseTest.restartMultiple();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_always_failed.yaml"})
|
||||
void restartFailedThenFailureWithGlobalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithGlobalErrors();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_local_errors.yaml"})
|
||||
void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
protected void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithLocalErrors();
|
||||
}
|
||||
|
||||
@@ -199,29 +195,27 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedWithAfterExecution();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
"flows/valids/trigger-flow-listener.yaml",
|
||||
"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-flow.yaml"})
|
||||
"flows/valids/trigger-flow.yaml"}, tenantId = "listener-tenant")
|
||||
void flowTrigger() throws Exception {
|
||||
flowTriggerCaseTest.trigger();
|
||||
flowTriggerCaseTest.trigger("listener-tenant");
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on CI but never fail locally
|
||||
@Test // flaky on CI but never fail locally
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-pause.yaml",
|
||||
"flows/valids/trigger-flow-with-pause.yaml"})
|
||||
void flowTriggerWithPause() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithPause();
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"})
|
||||
void flowTriggerWithConcurrencyLimit() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit();
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"}, tenantId = "trigger-tenant")
|
||||
protected void flowTriggerWithConcurrencyLimit() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit("trigger-tenant");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -232,12 +226,12 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.trigger();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
@Test // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-c.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"})
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"}, tenantId = "condition-tenant")
|
||||
void multipleConditionTriggerFailed() throws Exception {
|
||||
multipleConditionTriggerCaseTest.failed();
|
||||
multipleConditionTriggerCaseTest.failed("condition-tenant");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -248,12 +242,13 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditions();
|
||||
}
|
||||
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
@LoadFlows(value = {"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-a.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"})
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"}, tenantId = TENANT_1)
|
||||
void flowTriggerPreconditionsMergeOutputs() throws Exception {
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs();
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -262,7 +257,13 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.flowTriggerOnPaused();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-trigger-for-each-item-parent.yaml", "flows/valids/flow-trigger-for-each-item-child.yaml", "flows/valids/flow-trigger-for-each-item-grandchild.yaml"})
|
||||
void forEachItemWithFlowTrigger() throws Exception {
|
||||
multipleConditionTriggerCaseTest.forEachItemWithFlowTrigger();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/each-null.yaml"})
|
||||
void eachWithNull() throws Exception {
|
||||
EachSequentialTest.eachNullTest(runnerUtils, logsQueue);
|
||||
@@ -274,28 +275,28 @@ public abstract class AbstractRunnerTest {
|
||||
pluginDefaultsCaseTest.taskDefaults();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
void flowWaitSuccess() throws Exception {
|
||||
protected void flowWaitSuccess() throws Exception {
|
||||
flowCaseTest.waitSuccess();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_1)
|
||||
void flowWaitFailed() throws Exception {
|
||||
flowCaseTest.waitFailed();
|
||||
flowCaseTest.waitFailed(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_2)
|
||||
public void invalidOutputs() throws Exception {
|
||||
flowCaseTest.invalidOutputs();
|
||||
flowCaseTest.invalidOutputs(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -305,9 +306,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/working-directory.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/working-directory.yaml"}, tenantId = TENANT_1)
|
||||
public void workerFailed() throws Exception {
|
||||
workingDirectoryTest.failed(runnerUtils);
|
||||
workingDirectoryTest.failed(TENANT_1, runnerUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -322,7 +323,7 @@ public abstract class AbstractRunnerTest {
|
||||
workingDirectoryTest.cache(runnerUtils);
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on MySQL
|
||||
@Test // flaky on MySQL
|
||||
@LoadFlows({"flows/valids/pause-test.yaml"})
|
||||
public void pauseRun() throws Exception {
|
||||
pauseTest.run(runnerUtils);
|
||||
@@ -358,44 +359,44 @@ public abstract class AbstractRunnerTest {
|
||||
skipExecutionCaseTest.skipExecution();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item.yaml"})
|
||||
protected void forEachItem() throws Exception {
|
||||
forEachItemCaseTest.forEachItem();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item.yaml"})
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item.yaml"}, tenantId = TENANT_1)
|
||||
protected void forEachItemEmptyItems() throws Exception {
|
||||
forEachItemCaseTest.forEachItemEmptyItems();
|
||||
forEachItemCaseTest.forEachItemEmptyItems(TENANT_1);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow-failed.yaml",
|
||||
"flows/valids/for-each-item-failed.yaml"})
|
||||
protected void forEachItemFailed() throws Exception {
|
||||
forEachItemCaseTest.forEachItemFailed();
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/for-each-item-outputs-subflow.yaml",
|
||||
"flows/valids/for-each-item-outputs.yaml"})
|
||||
protected void forEachItemSubflowOutputs() throws Exception {
|
||||
forEachItemCaseTest.forEachItemWithSubflowOutputs();
|
||||
}
|
||||
|
||||
@RetryingTest(5) // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows({"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"})
|
||||
@Test // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows(value = {"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"}, tenantId = TENANT_1)
|
||||
void restartForEachItem() throws Exception {
|
||||
forEachItemCaseTest.restartForEachItem();
|
||||
forEachItemCaseTest.restartForEachItem(TENANT_1);
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"})
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"}, tenantId = TENANT_1)
|
||||
protected void forEachItemInIf() throws Exception {
|
||||
forEachItemCaseTest.forEachItemInIf();
|
||||
forEachItemCaseTest.forEachItemInIf(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -436,9 +437,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"})
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"}, tenantId = TENANT_1)
|
||||
protected void flowConcurrencyWithForEachItem() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem();
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -454,9 +455,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"})
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"}, tenantId = TENANT_1)
|
||||
void flowConcurrencySubflow() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencySubflow();
|
||||
flowConcurrencyCaseTest.flowConcurrencySubflow(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-parallel-subflow-kill.yaml", "flows/valids/flow-concurrency-parallel-subflow-kill-child.yaml", "flows/valids/flow-concurrency-parallel-subflow-kill-grandchild.yaml"})
|
||||
void flowConcurrencyParallelSubflowKill() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyParallelSubflowKill();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -511,9 +518,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/minimal.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/minimal.yaml"}, tenantId = TENANT_1)
|
||||
void shouldScheduleOnDate() throws Exception {
|
||||
scheduleDateCaseTest.shouldScheduleOnDate();
|
||||
scheduleDateCaseTest.shouldScheduleOnDate(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -535,15 +542,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_1)
|
||||
void executionConditionSLAShouldCancel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldCancel();
|
||||
slaTestCase.executionConditionSLAShouldCancel(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_2)
|
||||
void executionConditionSLAShouldLabel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldLabel();
|
||||
slaTestCase.executionConditionSLAShouldLabel(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -563,15 +570,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow("flows/valids/failed-first.yaml")
|
||||
@ExecuteFlow(value = "flows/valids/failed-first.yaml", tenantId = TENANT_1)
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
changeStateTestCase.changeStateShouldEndsInSuccess(execution);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"})
|
||||
@LoadFlows(value = {"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"}, tenantId = TENANT_2)
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess();
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess(TENANT_2);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -3,25 +3,18 @@ package io.kestra.core.runners;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Singleton
|
||||
public class ChangeStateTestCase {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests";
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@@ -29,11 +22,7 @@ public class ChangeStateTestCase {
|
||||
private ExecutionService executionService;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
private QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
@@ -41,73 +30,40 @@ public class ChangeStateTestCase {
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the last execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
Execution markedAs = executionService.markAs(execution, flow, execution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
executionQueue.emit(markedAs);
|
||||
Execution lastExecution = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), markedAs);
|
||||
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
// await for the subflow execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if ("failed-first".equals(exec.getFlowId()) && exec.getState().getCurrent() == State.Type.FAILED) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess(String tenantId) throws Exception {
|
||||
// run the parent flow
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-parent-of-failed");
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// assert on the subflow
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the parent execution
|
||||
CountDownLatch parentLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastParentExecution = new AtomicReference<>();
|
||||
receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().isTerminated()) {
|
||||
lastParentExecution.set(exec);
|
||||
parentLatch.countDown();
|
||||
}
|
||||
});
|
||||
Execution lastExecution = runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.FAILED), tenantId, NAMESPACE, "failed-first");
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// restart the subflow
|
||||
Flow flow = flowRepository.findByExecution(lastExecution.get());
|
||||
Execution markedAs = executionService.markAs(lastExecution.get(), flow, lastExecution.get().getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
executionQueue.emit(markedAs);
|
||||
Flow flow = flowRepository.findByExecution(lastExecution);
|
||||
Execution markedAs = executionService.markAs(lastExecution, flow, lastExecution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().isTerminated(), markedAs);
|
||||
|
||||
//We wait for the subflow execution to pass from failed to success
|
||||
Execution lastParentExecution = runnerUtils.awaitFlowExecution(e ->
|
||||
e.getTaskRunList().getFirst().getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
|
||||
// assert for the parent flow
|
||||
assertThat(parentLatch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastParentExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastParentExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
public class EmptyVariablesTest {
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
private TestRunnerUtils runnerUtils;
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user