mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-25 11:12:12 -05:00
Compare commits
114 Commits
executor_v
...
v1.0.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41468652d4 | ||
|
|
bc182277de | ||
|
|
8c2271089c | ||
|
|
9973a2120b | ||
|
|
bdfd038d40 | ||
|
|
a3fd734082 | ||
|
|
553a1d5389 | ||
|
|
c58aca967b | ||
|
|
27dcf60770 | ||
|
|
4e7c75232a | ||
|
|
f452da7ce1 | ||
|
|
43401c5017 | ||
|
|
067b110cf0 | ||
|
|
4ceff83a28 | ||
|
|
5026afe5bf | ||
|
|
3c899fcb2f | ||
|
|
cee412ffa9 | ||
|
|
3a57a683be | ||
|
|
a0b9de934e | ||
|
|
d677317cc5 | ||
|
|
9e661195e5 | ||
|
|
09c921bee5 | ||
|
|
d21ec4e899 | ||
|
|
efdb25fa97 | ||
|
|
37bdcc342c | ||
|
|
6d35f2b7a6 | ||
|
|
fe46ddf381 | ||
|
|
359dc9adc0 | ||
|
|
39c930124f | ||
|
|
1686fc3b4e | ||
|
|
03ff25ff55 | ||
|
|
d02fd53287 | ||
|
|
6c16bbe853 | ||
|
|
aa7a473d49 | ||
|
|
95133ebc40 | ||
|
|
54482e1d06 | ||
|
|
54b7811812 | ||
|
|
050ad60a95 | ||
|
|
030627ba7b | ||
|
|
c06ef7958f | ||
|
|
692d046289 | ||
|
|
92c1f04ec0 | ||
|
|
9e11d5fe5e | ||
|
|
14952c9457 | ||
|
|
ae314c301d | ||
|
|
f8aa5fb6ba | ||
|
|
c87d7e4da0 | ||
|
|
c928f1d822 | ||
|
|
baa07dd02b | ||
|
|
260cb50651 | ||
|
|
0a45325c69 | ||
|
|
c2522e2544 | ||
|
|
27476279ae | ||
|
|
3cc6372cb5 | ||
|
|
5f6e9dbe06 | ||
|
|
5078ce741d | ||
|
|
b7e17b7114 | ||
|
|
acaee34b0e | ||
|
|
1d78332505 | ||
|
|
7249632510 | ||
|
|
4a66a08c3b | ||
|
|
22fd6e97ea | ||
|
|
9afd86d32b | ||
|
|
797ea6c9e4 | ||
|
|
07d5e815c4 | ||
|
|
33ac9b1495 | ||
|
|
4d5b95d040 | ||
|
|
667aca7345 | ||
|
|
e05cc65202 | ||
|
|
71b606c27c | ||
|
|
47f9f12ce8 | ||
|
|
01acae5e97 | ||
|
|
e5878f08b7 | ||
|
|
0bcb6b4e0d | ||
|
|
3c2ecf4342 | ||
|
|
3d4f66772e | ||
|
|
e2afd4bcc3 | ||
|
|
d143097f03 | ||
|
|
72c0d91c1a | ||
|
|
1d692e56b0 | ||
|
|
0352d617ac | ||
|
|
b41aa4e0b9 | ||
|
|
d811dc030b | ||
|
|
105e62eee1 | ||
|
|
28796862a4 | ||
|
|
637cd794a4 | ||
|
|
fdd5c6e63d | ||
|
|
eda2483ec9 | ||
|
|
7b3c296489 | ||
|
|
fe6f8b4ed9 | ||
|
|
17ff539690 | ||
|
|
bbd0dda47e | ||
|
|
27a8e8b5a7 | ||
|
|
d6620a34cd | ||
|
|
6f8b3c5cfd | ||
|
|
6da6cbab60 | ||
|
|
a899e16178 | ||
|
|
568cd0b0c7 | ||
|
|
92e1dcb6eb | ||
|
|
499e040cd0 | ||
|
|
5916831d62 | ||
|
|
0b1b55957e | ||
|
|
7ee40d376a | ||
|
|
e2c9b3e256 | ||
|
|
556730777b | ||
|
|
c1a75a431f | ||
|
|
4a5b91667a | ||
|
|
f7b2af16a1 | ||
|
|
9351cb22e0 | ||
|
|
b1ecb82fdc | ||
|
|
c6d56151eb | ||
|
|
ed4398467a | ||
|
|
c51947419a | ||
|
|
ccb6a1f4a7 |
2
.github/ISSUE_TEMPLATE/bug.yml
vendored
2
.github/ISSUE_TEMPLATE/bug.yml
vendored
@@ -4,7 +4,7 @@ body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack). Don't forget to give us a star! ⭐
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack).
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Describe the issue
|
||||
|
||||
2
.github/ISSUE_TEMPLATE/feature.yml
vendored
2
.github/ISSUE_TEMPLATE/feature.yml
vendored
@@ -4,7 +4,7 @@ body:
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Feature description
|
||||
placeholder: Tell us more about your feature request. Don't forget to give us a star! ⭐
|
||||
placeholder: Tell us more about your feature request
|
||||
validations:
|
||||
required: true
|
||||
labels:
|
||||
|
||||
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@@ -35,4 +35,4 @@ Remove this section if this change applies to all flows or to the documentation
|
||||
|
||||
If there are no setup requirements, you can remove this section.
|
||||
|
||||
Thank you for your contribution. ❤️ Don't forget to give us a star! ⭐ -->
|
||||
Thank you for your contribution. ❤️ -->
|
||||
|
||||
67
.github/workflows/auto-translate-ui-keys.yml
vendored
67
.github/workflows/auto-translate-ui-keys.yml
vendored
@@ -1,67 +0,0 @@
|
||||
name: Auto-Translate UI keys and create PR
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9-21/3 * * *" # Every 3 hours from 9 AM to 9 PM
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
retranslate_modified_keys:
|
||||
description: "Whether to re-translate modified keys even if they already have translations."
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
default: "false"
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
translations:
|
||||
name: Translations
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
name: Checkout
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: pip install gitpython openai
|
||||
|
||||
- name: Generate translations
|
||||
run: python ui/src/translations/generate_translations.py ${{ github.event.inputs.retranslate_modified_keys }}
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: "20.x"
|
||||
|
||||
- name: Set up Git
|
||||
run: |
|
||||
git config --global user.name "GitHub Action"
|
||||
git config --global user.email "actions@github.com"
|
||||
|
||||
- name: Commit and create PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH_NAME="chore/update-translations-$(date +%s)"
|
||||
git checkout -b $BRANCH_NAME
|
||||
git add ui/src/translations/*.json
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit. Exiting with success."
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "chore(core): localize to languages other than english" -m "Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference."
|
||||
git push -u origin $BRANCH_NAME || (git push origin --delete $BRANCH_NAME && git push -u origin $BRANCH_NAME)
|
||||
gh pr create --title "Translations from en.json" --body $'This PR was created automatically by a GitHub Action.\n\nSomeone from the @kestra-io/frontend team needs to review and merge.' --base ${{ github.ref_name }} --head $BRANCH_NAME
|
||||
|
||||
- name: Check keys matching
|
||||
run: node ui/src/translations/check.js
|
||||
85
.github/workflows/codeql-analysis.yml
vendored
85
.github/workflows/codeql-analysis.yml
vendored
@@ -1,85 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 5 * * 1'
|
||||
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# Override automatic language detection by changing the below list
|
||||
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
|
||||
language: ['java', 'javascript']
|
||||
# Learn more...
|
||||
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
# We must fetch at least the immediate parents so that if this is
|
||||
# a pull request then we can checkout the head.
|
||||
fetch-depth: 2
|
||||
|
||||
# If this run was triggered by a pull request event, then checkout
|
||||
# the head of the pull request instead of the merge commit.
|
||||
- run: git checkout HEAD^2
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Set up JDK
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: 21
|
||||
|
||||
- name: Setup gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
|
||||
- name: Build with Gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
run: ./gradlew testClasses -x :ui:assembleFrontend
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
if: ${{ matrix.language != 'java' }}
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
15
.github/workflows/e2e-scheduling.yml
vendored
15
.github/workflows/e2e-scheduling.yml
vendored
@@ -1,15 +0,0 @@
|
||||
name: 'E2E tests scheduling'
|
||||
# 'New E2E tests implementation started by Roman. Based on playwright in npm UI project, tests Kestra OSS develop docker image. These tests are written from zero, lets make them unflaky from the start!.'
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 * * * *" # Every hour
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
jobs:
|
||||
e2e:
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-e2e-tests.yml@main
|
||||
@@ -1,84 +0,0 @@
|
||||
name: Create new release branch
|
||||
run-name: "Create new release branch Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
nextVersion:
|
||||
description: 'The next version (e.g., 0.22.0-SNAPSHOT)'
|
||||
required: true
|
||||
type: string
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
NEXT_VERSION: "${{ github.event.inputs.nextVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
steps:
|
||||
# Checks
|
||||
- name: Check Inputs
|
||||
run: |
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)\.0$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$NEXT_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$ ]]; then
|
||||
echo "Invalid next version. Must match regex: ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$"
|
||||
exit 1;
|
||||
fi
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: kestra
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
caches-enabled: true
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
PUSH_RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
|
||||
cd kestra
|
||||
|
||||
# Create and push release branch
|
||||
git checkout -b "$PUSH_RELEASE_BRANCH";
|
||||
git push -u origin "$PUSH_RELEASE_BRANCH";
|
||||
|
||||
# Run gradle release
|
||||
git checkout develop;
|
||||
|
||||
if [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]]; then
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}" \
|
||||
-Prelease.failOnSnapshotDependencies=false
|
||||
else
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}"
|
||||
fi
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Run Gradle Release for Kestra Plugins
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
nextVersion:
|
||||
description: 'The next version (e.g., 0.22.0-SNAPSHOT)'
|
||||
required: true
|
||||
type: string
|
||||
dryRun:
|
||||
description: 'Use DRY_RUN mode'
|
||||
required: false
|
||||
default: 'false'
|
||||
jobs:
|
||||
release:
|
||||
name: Release plugins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
|
||||
- name: 'Configure Git'
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
if: ${{ github.event.inputs.dryRun == 'false' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
|
||||
- name: Run Gradle Release (DRY_RUN)
|
||||
if: ${{ github.event.inputs.dryRun == 'true' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
--dry-run \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
@@ -1,60 +0,0 @@
|
||||
name: Set Version and Tag Plugins
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
dryRun:
|
||||
description: 'Use DRY_RUN mode'
|
||||
required: false
|
||||
default: 'false'
|
||||
jobs:
|
||||
tag:
|
||||
name: Release plugins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
|
||||
- name: 'Configure Git'
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Set Version and Tag Plugins
|
||||
if: ${{ github.event.inputs.dryRun == 'false' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
|
||||
- name: Set Version and Tag Plugins (DRY_RUN)
|
||||
if: ${{ github.event.inputs.dryRun == 'true' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--dry-run \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
65
.github/workflows/global-start-release.yml
vendored
65
.github/workflows/global-start-release.yml
vendored
@@ -1,65 +0,0 @@
|
||||
name: Start release
|
||||
run-name: "Start release of Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.1)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Parse and Check Inputs
|
||||
id: parse-and-check-inputs
|
||||
run: |
|
||||
CURRENT_BRANCH="${{ github.ref_name }}"
|
||||
if ! [[ "$CURRENT_BRANCH" == "develop" ]]; then
|
||||
echo "You can only run this workflow on develop, but you ran it on $CURRENT_BRANCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)(\.[0-9]+)(-rc[0-9])?(-SNAPSHOT)?$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
echo "release_branch=${RELEASE_BRANCH}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Checkout
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
ref: ${{ steps.parse-and-check-inputs.outputs.release_branch }}
|
||||
|
||||
# Configure
|
||||
- name: Git - Configure
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Start release by updating version and pushing a new tag
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Update version
|
||||
sed -i "s/^version=.*/version=$RELEASE_VERSION/" ./gradle.properties
|
||||
git add ./gradle.properties
|
||||
git commit -m"chore(version): update to version '$RELEASE_VERSION'"
|
||||
git push
|
||||
git tag -a "v$RELEASE_VERSION" -m"v$RELEASE_VERSION"
|
||||
git push --tags
|
||||
9
.github/workflows/main-build.yml
vendored
9
.github/workflows/main-build.yml
vendored
@@ -67,13 +67,12 @@ jobs:
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [backend-tests, frontend-tests, publish-develop-docker, publish-develop-maven]
|
||||
needs: [publish-develop-docker, publish-develop-maven]
|
||||
if: always()
|
||||
steps:
|
||||
- run: echo "debug repo ${{github.repository}} ref ${{github.ref}} res ${{needs.publish-develop-maven.result}} jobStatus ${{job.status}} isNotFork ${{github.repository == 'kestra-io/kestra'}} isDevelop ${{github.ref == 'refs/heads/develop'}}"
|
||||
- name: Trigger EE Workflow
|
||||
uses: peter-evans/repository-dispatch@v4
|
||||
if: github.ref == 'refs/heads/develop' && needs.publish-develop-maven == 'success'
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
if: github.ref == 'refs/heads/develop' && needs.release.result == 'success'
|
||||
with:
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
repository: kestra-io/kestra-ee
|
||||
@@ -81,7 +80,7 @@ jobs:
|
||||
|
||||
# Slack
|
||||
- name: Slack - Notification
|
||||
if: ${{ failure() && github.repository == 'kestra-io/kestra' && (github.ref == 'refs/heads/develop') }}
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
74
.github/workflows/vulnerabilities-check.yml
vendored
74
.github/workflows/vulnerabilities-check.yml
vendored
@@ -48,77 +48,3 @@ jobs:
|
||||
with:
|
||||
name: dependency-check-report
|
||||
path: build/reports/dependency-check-report.html
|
||||
|
||||
develop-image-check:
|
||||
name: Image Check (develop)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:develop
|
||||
format: 'template'
|
||||
template: '@/contrib/sarif.tpl'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
skip-dirs: /app/plugins
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
|
||||
latest-image-check:
|
||||
name: Image Check (latest)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:latest
|
||||
format: table
|
||||
skip-dirs: /app/plugins
|
||||
scanners: vuln
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
13
README.md
13
README.md
@@ -19,12 +19,9 @@
|
||||
<br />
|
||||
|
||||
<p align="center">
|
||||
<a href="https://twitter.com/kestra_io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/twitter.svg" alt="twitter" width="35" height="25" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" width="35" height="25" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/youtube.svg" alt="youtube" width="35" height="25" /></a>
|
||||
<a href="https://x.com/kestra_io"><img height="25" src="https://kestra.io/twitter.svg" alt="X(formerly Twitter)" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/"><img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io"><img height="25" src="https://kestra.io/youtube.svg" alt="youtube" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -36,10 +33,10 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.kestra.io/video/product-overview" target="_blank">
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
|
||||
</a>
|
||||
</p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
|
||||
|
||||
|
||||
## 🌟 What is Kestra?
|
||||
|
||||
83
build.gradle
83
build.gradle
@@ -25,19 +25,19 @@ plugins {
|
||||
id 'jacoco-report-aggregation'
|
||||
|
||||
// helper
|
||||
id "com.github.ben-manes.versions" version "0.53.0"
|
||||
id "com.github.ben-manes.versions" version "0.52.0"
|
||||
|
||||
// front
|
||||
id 'com.github.node-gradle.node' version '7.1.0'
|
||||
|
||||
// release
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.3"
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.2"
|
||||
id 'signing'
|
||||
id "com.vanniktech.maven.publish" version "0.34.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.6" apply false
|
||||
id "org.owasp.dependencycheck" version "12.1.3" apply false
|
||||
}
|
||||
|
||||
idea {
|
||||
@@ -168,9 +168,8 @@ allprojects {
|
||||
/**********************************************************************************************************************\
|
||||
* Test
|
||||
**********************************************************************************************************************/
|
||||
subprojects {subProj ->
|
||||
|
||||
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
|
||||
subprojects {
|
||||
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
|
||||
apply plugin: "com.adarshr.test-logger"
|
||||
|
||||
java {
|
||||
@@ -206,67 +205,23 @@ subprojects {subProj ->
|
||||
testImplementation 'org.assertj:assertj-core'
|
||||
}
|
||||
|
||||
def commonTestConfig = { Test t ->
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
|
||||
// set Xmx for test workers
|
||||
t.maxHeapSize = '4g'
|
||||
maxHeapSize = '4g'
|
||||
|
||||
// configure en_US default locale for tests
|
||||
t.systemProperty 'user.language', 'en'
|
||||
t.systemProperty 'user.country', 'US'
|
||||
systemProperty 'user.language', 'en'
|
||||
systemProperty 'user.country', 'US'
|
||||
|
||||
t.environment 'SECRET_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
|
||||
t.environment 'SECRET_NEW_LINE', "cGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2\nZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZl\neXJsb25n"
|
||||
t.environment 'SECRET_WEBHOOK_KEY', "secretKey".bytes.encodeBase64().toString()
|
||||
t.environment 'SECRET_NON_B64_SECRET', "some secret value"
|
||||
t.environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
|
||||
t.environment 'ENV_TEST1', "true"
|
||||
t.environment 'ENV_TEST2', "Pass by env"
|
||||
|
||||
|
||||
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
|
||||
// JUnit 5 parallel settings
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('flakyTest', Test) { Test t ->
|
||||
group = 'verification'
|
||||
description = 'Runs tests tagged @Flaky but does not fail the build.'
|
||||
|
||||
useJUnitPlatform {
|
||||
includeTags 'flaky'
|
||||
}
|
||||
ignoreFailures = true
|
||||
|
||||
reports {
|
||||
junitXml.required = true
|
||||
junitXml.outputPerTestCase = true
|
||||
junitXml.mergeReruns = true
|
||||
junitXml.includeSystemErrLog = true
|
||||
junitXml.outputLocation = layout.buildDirectory.dir("test-results/flakyTest")
|
||||
}
|
||||
commonTestConfig(t)
|
||||
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform {
|
||||
excludeTags 'flaky'
|
||||
}
|
||||
reports {
|
||||
junitXml.required = true
|
||||
junitXml.outputPerTestCase = true
|
||||
junitXml.mergeReruns = true
|
||||
junitXml.includeSystemErrLog = true
|
||||
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
|
||||
}
|
||||
commonTestConfig(it)
|
||||
|
||||
|
||||
finalizedBy(tasks.named('flakyTest'))
|
||||
environment 'SECRET_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
|
||||
environment 'SECRET_NEW_LINE', "cGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2\nZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZl\neXJsb25n"
|
||||
environment 'SECRET_WEBHOOK_KEY', "secretKey".bytes.encodeBase64().toString()
|
||||
environment 'SECRET_NON_B64_SECRET', "some secret value"
|
||||
environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
|
||||
environment 'ENV_TEST1', "true"
|
||||
environment 'ENV_TEST2', "Pass by env"
|
||||
}
|
||||
|
||||
testlogger {
|
||||
@@ -372,7 +327,7 @@ tasks.named('testCodeCoverageReport') {
|
||||
subprojects {
|
||||
sonar {
|
||||
properties {
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml,$projectDir.parentFile.path/build/reports/jacoco/test//testCodeCoverageReport.xml"
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,5 @@ dependencies {
|
||||
implementation project(":worker")
|
||||
|
||||
//test
|
||||
testImplementation project(':tests')
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
@@ -43,13 +43,13 @@ import java.util.concurrent.Callable;
|
||||
SysCommand.class,
|
||||
ConfigCommand.class,
|
||||
NamespaceCommand.class,
|
||||
MigrationCommand.class
|
||||
MigrationCommand.class,
|
||||
}
|
||||
)
|
||||
@Introspected
|
||||
public class App implements Callable<Integer> {
|
||||
public static void main(String[] args) {
|
||||
execute(App.class, new String [] { Environment.CLI }, args);
|
||||
execute(App.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
|
||||
return PicocliRunner.call(App.class, "--help");
|
||||
}
|
||||
|
||||
protected static void execute(Class<?> cls, String[] environments, String... args) {
|
||||
protected static void execute(Class<?> cls, String... args) {
|
||||
// Log Bridge
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
|
||||
// Init ApplicationContext
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, args);
|
||||
|
||||
// Call Picocli command
|
||||
int exitCode = 0;
|
||||
@@ -80,7 +80,6 @@ public class App implements Callable<Integer> {
|
||||
System.exit(Objects.requireNonNullElse(exitCode, 0));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
|
||||
* forced Properties from current command.
|
||||
@@ -89,13 +88,12 @@ public class App implements Callable<Integer> {
|
||||
* @return the application context created
|
||||
*/
|
||||
protected static ApplicationContext applicationContext(Class<?> mainClass,
|
||||
String[] environments,
|
||||
String[] args) {
|
||||
|
||||
ApplicationContextBuilder builder = ApplicationContext
|
||||
.builder()
|
||||
.mainClass(mainClass)
|
||||
.environments(environments);
|
||||
.environments(Environment.CLI);
|
||||
|
||||
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
|
||||
continueOnParsingErrors(cmd);
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.runners.*;
|
||||
import io.kestra.core.server.Service;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.ExecutorsUtils;
|
||||
import io.kestra.executor.DefaultExecutor;
|
||||
import io.kestra.worker.DefaultWorker;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.context.annotation.Value;
|
||||
@@ -50,7 +49,7 @@ public class StandAloneRunner implements Runnable, AutoCloseable {
|
||||
running.set(true);
|
||||
|
||||
poolExecutor = executorsUtils.cachedThreadPool("standalone-runner");
|
||||
poolExecutor.execute(applicationContext.getBean(DefaultExecutor.class));
|
||||
poolExecutor.execute(applicationContext.getBean(ExecutorInterface.class));
|
||||
|
||||
if (workerEnabled) {
|
||||
// FIXME: For backward-compatibility with Kestra 0.15.x and earliest we still used UUID for Worker ID instead of IdUtils
|
||||
|
||||
@@ -2,27 +2,19 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
|
||||
Integer serverPort;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
|
||||
|
||||
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
|
||||
|
||||
return super.call();
|
||||
}
|
||||
|
||||
private long maxMemoryInMB() {
|
||||
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
|
||||
}
|
||||
|
||||
protected static int defaultWorkerThread() {
|
||||
return Runtime.getRuntime().availableProcessors() * 8;
|
||||
}
|
||||
|
||||
@@ -262,8 +262,6 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private String getTenantIdFromPath(Path path) {
|
||||
// FIXME there is probably a bug here when a tenant has '_' in its name,
|
||||
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
|
||||
return path.getFileName().toString().split("_")[0];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,8 +49,6 @@ micronaut:
|
||||
- /ui/.+
|
||||
- /health
|
||||
- /health/.+
|
||||
- /metrics
|
||||
- /metrics/.+
|
||||
- /prometheus
|
||||
http-version: HTTP_1_1
|
||||
caches:
|
||||
|
||||
@@ -37,7 +37,7 @@ class AppTest {
|
||||
|
||||
final String[] args = new String[]{"server", serverType, "--help"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
|
||||
|
||||
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
|
||||
@@ -52,7 +52,7 @@ class AppTest {
|
||||
|
||||
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
|
||||
|
||||
assertThat(out.toString()).startsWith("Missing required parameters: ");
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package io.kestra.cli.commands.configs.sys;
|
||||
import io.kestra.cli.commands.flows.FlowCreateCommand;
|
||||
import io.kestra.cli.commands.namespaces.kv.KvCommand;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.runtime.server.EmbeddedServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
/**
|
||||
* Verifies CLI behavior without repository configuration:
|
||||
* - Repo-independent commands succeed (e.g. KV with no params).
|
||||
* - Repo-dependent commands fail with a clear error.
|
||||
*/
|
||||
class NoConfigCommandTest {
|
||||
|
||||
@Test
|
||||
void shouldSucceedWithNamespaceKVCommandWithoutParamsAndConfig() {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
System.setOut(new PrintStream(out));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
|
||||
String[] args = {};
|
||||
Integer call = PicocliRunner.call(KvCommand.class, ctx, args);
|
||||
|
||||
assertThat(call).isZero();
|
||||
assertThat(out.toString()).contains("Usage: kestra namespace kv");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailWithCreateFlowCommandWithoutConfig() throws URISyntaxException {
|
||||
URL flowUrl = NoConfigCommandTest.class.getClassLoader().getResource("crudFlow/date.yml");
|
||||
Objects.requireNonNull(flowUrl, "Test flow resource not found");
|
||||
|
||||
Path flowPath = Paths.get(flowUrl.toURI());
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
ByteArrayOutputStream err=new ByteArrayOutputStream();
|
||||
|
||||
System.setOut(new PrintStream(out));
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder()
|
||||
.deduceEnvironment(false)
|
||||
.start()) {
|
||||
|
||||
EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
|
||||
embeddedServer.start();
|
||||
|
||||
String[] createArgs = {
|
||||
"--server",
|
||||
embeddedServer.getURL().toString(),
|
||||
"--user",
|
||||
"myuser:pass:word",
|
||||
flowPath.toString(),
|
||||
};
|
||||
|
||||
Integer exitCode = PicocliRunner.call(FlowCreateCommand.class, ctx, createArgs);
|
||||
|
||||
|
||||
assertThat(exitCode).isNotZero();
|
||||
assertThat(out.toString()).isEmpty();
|
||||
assertThat(err.toString()).contains("No bean of type [io.kestra.core.repositories.FlowRepositoryInterface] exists");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,15 +1,14 @@
|
||||
package io.kestra.cli.services;
|
||||
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -19,8 +18,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -58,12 +57,10 @@ class FileChangedEventListenerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void test() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a basic flow
|
||||
String flow = """
|
||||
@@ -76,14 +73,14 @@ class FileChangedEventListenerTest {
|
||||
message: Hello World! 🚀
|
||||
""";
|
||||
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
assertThat(myflow.getTasks()).hasSize(1);
|
||||
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
|
||||
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -91,18 +88,16 @@ class FileChangedEventListenerTest {
|
||||
// delete the flow
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void testWithPluginDefault() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a flow with plugin default
|
||||
String pluginDefault = """
|
||||
@@ -118,14 +113,14 @@ class FileChangedEventListenerTest {
|
||||
values:
|
||||
message: Hello World!
|
||||
""";
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -133,7 +128,7 @@ class FileChangedEventListenerTest {
|
||||
// delete both files
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
|
||||
@@ -84,7 +84,7 @@ dependencies {
|
||||
|
||||
testImplementation "org.testcontainers:testcontainers:1.21.3"
|
||||
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
|
||||
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Schema(description = "A key/value pair that can be attached to a Flow or Execution. Labels are often used to organize and categorize objects.")
|
||||
@@ -44,7 +43,7 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
@@ -59,7 +58,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
@@ -74,7 +72,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
if (map == null || map.isEmpty()) return List.of();
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
@@ -93,14 +90,4 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides predicate for not empty entries.
|
||||
*
|
||||
* @return The non-empty filter
|
||||
*/
|
||||
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
|
||||
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
|
||||
entry.getValue() != null && !entry.getValue().isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,16 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface that can be implemented by classes supporting plugin versioning.
|
||||
*
|
||||
* @see Plugin
|
||||
*/
|
||||
public interface PluginVersioning {
|
||||
|
||||
String TITLE = "Plugin Version";
|
||||
String DESCRIPTION = """
|
||||
Defines the version of the plugin to use.
|
||||
|
||||
The version must follow the Semantic Versioning (SemVer) specification:
|
||||
- A single-digit MAJOR version (e.g., `1`).
|
||||
- A MAJOR.MINOR version (e.g., `1.1`).
|
||||
- A MAJOR.MINOR.PATCH version, optionally with any qualifier
|
||||
(e.g., `1.1.2`, `1.1.0-SNAPSHOT`).
|
||||
""";
|
||||
|
||||
@Schema(
|
||||
title = TITLE,
|
||||
description = DESCRIPTION
|
||||
)
|
||||
@Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)")
|
||||
@Schema(title = "The version of the plugin to use.")
|
||||
String getVersion();
|
||||
}
|
||||
|
||||
@@ -91,12 +91,6 @@ public record QueryFilter(
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX, Op.IN, Op.NOT_IN, Op.PREFIX);
|
||||
}
|
||||
},
|
||||
KIND("kind") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.EQUALS,Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
LABELS("labels") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
@@ -217,7 +211,7 @@ public record QueryFilter(
|
||||
return List.of(
|
||||
Field.QUERY, Field.SCOPE, Field.FLOW_ID, Field.START_DATE, Field.END_DATE,
|
||||
Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER,
|
||||
Field.NAMESPACE,Field.KIND
|
||||
Field.NAMESPACE
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -260,6 +254,18 @@ public record QueryFilter(
|
||||
*
|
||||
* @return List of {@code ResourceField} with resource names, fields, and operations.
|
||||
*/
|
||||
public static List<ResourceField> asResourceList() {
|
||||
return Arrays.stream(values())
|
||||
.map(Resource::toResourceField)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ResourceField toResourceField(Resource resource) {
|
||||
List<FieldOp> fieldOps = resource.supportedField().stream()
|
||||
.map(Resource::toFieldInfo)
|
||||
.toList();
|
||||
return new ResourceField(resource.name().toLowerCase(), fieldOps);
|
||||
}
|
||||
|
||||
private static FieldOp toFieldInfo(Field field) {
|
||||
List<Operation> operations = field.supportedOp().stream()
|
||||
@@ -273,6 +279,9 @@ public record QueryFilter(
|
||||
}
|
||||
}
|
||||
|
||||
public record ResourceField(String name, List<FieldOp> fields) {
|
||||
}
|
||||
|
||||
public record FieldOp(String name, String value, List<Operation> operations) {
|
||||
}
|
||||
|
||||
|
||||
@@ -17,12 +17,31 @@ import java.util.List;
|
||||
@Introspected
|
||||
public class ExecutionUsage {
|
||||
private final List<DailyExecutionStatistics> dailyExecutionsCount;
|
||||
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
|
||||
|
||||
public static ExecutionUsage of(final String tenantId,
|
||||
final ExecutionRepositoryInterface executionRepository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
|
||||
try {
|
||||
dailyTaskRunsCount = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenantId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
true);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
|
||||
}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(executionRepository.dailyStatistics(
|
||||
null,
|
||||
@@ -33,13 +52,28 @@ public class ExecutionUsage {
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null))
|
||||
null,
|
||||
false))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
try {
|
||||
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
true
|
||||
);
|
||||
} catch (UnsupportedOperationException ignored) {}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
@@ -47,8 +81,10 @@ public class ExecutionUsage {
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY
|
||||
DateUtils.GroupType.DAY,
|
||||
false
|
||||
))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,7 +496,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
if (resolvedFinally != null && (
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailedNoRetry(resolvedTasks, parentTaskRun
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailed(resolvedTasks, parentTaskRun
|
||||
))) {
|
||||
return resolvedFinally;
|
||||
}
|
||||
@@ -584,13 +584,6 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastSubmitted(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
.filter(t -> t.getState().getCurrent() == State.Type.SUBMITTED)
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastRunning(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
@@ -872,18 +865,20 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
* @param e the exception raise
|
||||
* @return new taskRun with updated attempt with logs
|
||||
*/
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
|
||||
TaskRun failed = taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
|
||||
TaskRunAttempt lastAttempt, Exception e) {
|
||||
return new FailedTaskRunWithLog(
|
||||
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
|
||||
taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt
|
||||
.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
)
|
||||
.withState(State.Type.FAILED),
|
||||
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -296,7 +296,7 @@ public class TaskRun implements TenantInterface {
|
||||
}
|
||||
|
||||
public TaskRun incrementIteration() {
|
||||
int iteration = this.iteration == null ? 0 : this.iteration;
|
||||
int iteration = this.iteration == null ? 1 : this.iteration;
|
||||
return this.toBuilder()
|
||||
.iteration(iteration + 1)
|
||||
.build();
|
||||
|
||||
@@ -222,7 +222,6 @@ public class State {
|
||||
@Introspected
|
||||
public enum Type {
|
||||
CREATED,
|
||||
SUBMITTED,
|
||||
RUNNING,
|
||||
PAUSED,
|
||||
RESTARTED,
|
||||
|
||||
@@ -30,7 +30,7 @@ public class ResolvedTask {
|
||||
|
||||
public NextTaskRun toNextTaskRunIncrementIteration(Execution execution, Integer iteration) {
|
||||
return new NextTaskRun(
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 0),
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 1),
|
||||
this.getTask()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,11 +1,3 @@
|
||||
package io.kestra.core.models.tasks.runners;
|
||||
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
public interface RemoteRunnerInterface {
|
||||
@Schema(
|
||||
title = "Whether to synchronize working directory from remote runner back to local one after run."
|
||||
)
|
||||
Property<Boolean> getSyncWorkingDirectory();
|
||||
}
|
||||
public interface RemoteRunnerInterface {}
|
||||
|
||||
@@ -30,10 +30,6 @@ public interface TaskCommands {
|
||||
|
||||
Map<String, Object> getAdditionalVars();
|
||||
|
||||
default String outputDirectoryName() {
|
||||
return this.getWorkingDirectory().relativize(this.getOutputDirectory()).toString();
|
||||
}
|
||||
|
||||
Path getWorkingDirectory();
|
||||
|
||||
Path getOutputDirectory();
|
||||
|
||||
@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
*
|
||||
* @return the {@link DefaultPluginRegistry}.
|
||||
*/
|
||||
public synchronized static DefaultPluginRegistry getOrCreate() {
|
||||
public static DefaultPluginRegistry getOrCreate() {
|
||||
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
|
||||
if (!instance.isInitialized()) {
|
||||
instance.init();
|
||||
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
/**
|
||||
* Initializes the registry by loading all core plugins.
|
||||
*/
|
||||
protected synchronized void init() {
|
||||
protected void init() {
|
||||
if (initialized.compareAndSet(false, true)) {
|
||||
register(scanner.scan());
|
||||
}
|
||||
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
if (existing != null && existing.crc32() == plugin.crc32()) {
|
||||
return; // same plugin already registered
|
||||
}
|
||||
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
if (existing != null) {
|
||||
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
|
||||
pluginClassByIdentifier.putAll(plugins);
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
|
||||
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
|
||||
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::asText).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
|
||||
|
||||
if (type == null || type.isEmpty()) {
|
||||
return null;
|
||||
|
||||
@@ -11,7 +11,6 @@ import io.kestra.core.runners.*;
|
||||
|
||||
public interface QueueFactoryInterface {
|
||||
String EXECUTION_NAMED = "executionQueue";
|
||||
String EXECUTION_EVENT_NAMED = "executionEventQueue";
|
||||
String EXECUTOR_NAMED = "executorQueue";
|
||||
String WORKERJOB_NAMED = "workerJobQueue";
|
||||
String WORKERTASKRESULT_NAMED = "workerTaskResultQueue";
|
||||
@@ -31,8 +30,6 @@ public interface QueueFactoryInterface {
|
||||
|
||||
QueueInterface<Execution> execution();
|
||||
|
||||
QueueInterface<ExecutionEvent> executionEvent();
|
||||
|
||||
QueueInterface<Executor> executor();
|
||||
|
||||
WorkerJobQueueInterface workerJob();
|
||||
|
||||
@@ -35,24 +35,6 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
|
||||
|
||||
void delete(String consumerGroup, T message) throws QueueException;
|
||||
|
||||
/**
|
||||
* Delete all messages of the queue for this key.
|
||||
* This is used to purge a queue for a specific key.
|
||||
* A queue implementation may omit to implement it and purge records differently.
|
||||
*/
|
||||
default void deleteByKey(String key) throws QueueException {
|
||||
// by default do nothing
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all messages of the queue for a set of keys.
|
||||
* This is used to purge a queue for specific keys.
|
||||
* A queue implementation may omit to implement it and purge records differently.
|
||||
*/
|
||||
default void deleteByKeys(List<String> keys) throws QueueException {
|
||||
// by default do nothing
|
||||
}
|
||||
|
||||
default Runnable receive(Consumer<Either<T, DeserializationException>> consumer) {
|
||||
return receive(null, consumer, false);
|
||||
}
|
||||
@@ -72,20 +54,4 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
|
||||
}
|
||||
|
||||
Runnable receive(String consumerGroup, Class<?> queueType, Consumer<Either<T, DeserializationException>> consumer, boolean forUpdate);
|
||||
|
||||
default Runnable receiveBatch(Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
|
||||
return receiveBatch(null, queueType, consumer);
|
||||
}
|
||||
|
||||
default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
|
||||
return receiveBatch(consumerGroup, queueType, consumer, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumer a batch of messages.
|
||||
* By default, it consumes a single message, a queue implementation may implement it to support batch consumption.
|
||||
*/
|
||||
default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer, boolean forUpdate) {
|
||||
return receive(consumerGroup, either -> consumer.accept(List.of(either)), forUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,8 @@ public class QueueService {
|
||||
return ((SubflowExecution<?>) object).getExecution().getId();
|
||||
} else if (object.getClass() == SubflowExecutionResult.class) {
|
||||
return ((SubflowExecutionResult) object).getExecutionId();
|
||||
} else if (object.getClass() == ExecutorState.class) {
|
||||
return ((ExecutorState) object).getExecutionId();
|
||||
} else if (object.getClass() == Setting.class) {
|
||||
return ((Setting) object).getKey();
|
||||
} else if (object.getClass() == Executor.class) {
|
||||
|
||||
@@ -2,12 +2,12 @@ package io.kestra.core.repositories;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.executions.statistics.DailyExecutionStatistics;
|
||||
import io.kestra.core.models.executions.statistics.ExecutionCount;
|
||||
import io.kestra.core.models.executions.statistics.Flow;
|
||||
import io.kestra.core.models.flows.FlowScope;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.runners.Executor;
|
||||
import io.kestra.core.utils.DateUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
@@ -25,6 +25,8 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
|
||||
Boolean isTaskRunEnabled();
|
||||
|
||||
default Optional<Execution> findById(String tenantId, String id) {
|
||||
return findById(tenantId, id, false);
|
||||
}
|
||||
@@ -94,6 +96,12 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
|
||||
Flux<Execution> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
ArrayListTotal<TaskRun> findTaskRun(
|
||||
Pageable pageable,
|
||||
@Nullable String tenantId,
|
||||
List<QueryFilter> filters
|
||||
);
|
||||
|
||||
Execution delete(Execution execution);
|
||||
|
||||
Integer purge(Execution execution);
|
||||
@@ -106,7 +114,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatistics(
|
||||
@@ -118,7 +127,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
List<State.Type> state
|
||||
List<State.Type> state,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
@Getter
|
||||
@@ -156,6 +166,4 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
String tenantId,
|
||||
@Nullable List<FlowFilter> flows
|
||||
);
|
||||
|
||||
Executor lock(String executionId, Function<Execution, Executor> function);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.plugin.core.dashboard.data.Flows;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -12,7 +11,7 @@ import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fields> {
|
||||
public interface FlowRepositoryInterface {
|
||||
|
||||
Optional<Flow> findById(String tenantId, String namespace, String id, Optional<Integer> revision, Boolean allowDeleted);
|
||||
|
||||
@@ -163,6 +162,4 @@ public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fie
|
||||
FlowWithSource update(GenericFlow flow, FlowInterface previous) throws ConstraintViolationException;
|
||||
|
||||
FlowWithSource delete(FlowInterface flow);
|
||||
|
||||
Boolean existAnyNoAcl(String tenantId);
|
||||
}
|
||||
|
||||
@@ -83,9 +83,7 @@ public class LocalFlowRepositoryLoader {
|
||||
}
|
||||
|
||||
public void load(String tenantId, File basePath) throws IOException {
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
|
||||
.stream()
|
||||
.filter(flow -> tenantId.equals(flow.getTenantId()))
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
|
||||
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
|
||||
|
||||
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {
|
||||
|
||||
@@ -5,7 +5,10 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.exceptions.InternalException;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
@@ -26,7 +29,6 @@ import org.apache.commons.lang3.stream.Streams;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.trace.Tracer.throwCallable;
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
@@ -151,22 +153,14 @@ public final class ExecutableUtils {
|
||||
currentFlow.getNamespace(),
|
||||
currentFlow.getId()
|
||||
)
|
||||
.orElseThrow(() -> {
|
||||
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
|
||||
runContext.logger().error(msg);
|
||||
return new IllegalStateException(msg);
|
||||
});
|
||||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
|
||||
|
||||
if (flow.isDisabled()) {
|
||||
String msg = "Cannot execute a flow which is disabled";
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute a flow which is disabled");
|
||||
}
|
||||
|
||||
if (flow instanceof FlowWithException fwe) {
|
||||
String msg = "Cannot execute an invalid flow: " + fwe.getException();
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
|
||||
}
|
||||
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
@@ -207,20 +201,7 @@ public final class ExecutableUtils {
|
||||
.build()
|
||||
)
|
||||
.withScheduleDate(scheduleOnDate);
|
||||
if(execution.getInputs().size()<inputs.size()) {
|
||||
Map<String,Object>resolvedInputs=execution.getInputs();
|
||||
for (var inputKey : inputs.keySet()) {
|
||||
if (!resolvedInputs.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
|
||||
inputKey,
|
||||
currentExecution.getId(),
|
||||
currentTask.subflowId().namespace(),
|
||||
currentTask.subflowId().flowId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inject the traceparent into the new execution
|
||||
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
public record ExecutionEvent(String tenantId, String executionId, Instant eventDate, ExecutionEventType eventType) implements HasUID {
|
||||
public ExecutionEvent(Execution execution, ExecutionEventType eventType) {
|
||||
this(execution.getTenantId(), execution.getId(), Instant.now(), eventType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String uid() {
|
||||
return executionId;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
public enum ExecutionEventType {
|
||||
CREATED,
|
||||
UPDATED,
|
||||
TERMINATED,
|
||||
}
|
||||
21
core/src/main/java/io/kestra/core/runners/ExecutorState.java
Normal file
21
core/src/main/java/io/kestra/core/runners/ExecutorState.java
Normal file
@@ -0,0 +1,21 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.flows.State;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class ExecutorState {
|
||||
private String executionId;
|
||||
private Map<String, State.Type> workerTaskDeduplication = new ConcurrentHashMap<>();
|
||||
private Map<String, String> childDeduplication = new ConcurrentHashMap<>();
|
||||
private Map<String, State.Type> subflowExecutionDeduplication = new ConcurrentHashMap<>();
|
||||
|
||||
public ExecutorState(String executionId) {
|
||||
this.executionId = executionId;
|
||||
}
|
||||
}
|
||||
@@ -82,8 +82,6 @@ public abstract class FilesService {
|
||||
}
|
||||
|
||||
private static String resolveUniqueNameForFile(final Path path) {
|
||||
String filename = path.getFileName().toString();
|
||||
String encodedFilename = java.net.URLEncoder.encode(filename, java.nio.charset.StandardCharsets.UTF_8);
|
||||
return IdUtils.from(path.toString()) + "-" + encodedFilename;
|
||||
return IdUtils.from(path.toString()) + "-" + path.toFile().getName();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,15 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.util.*;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -220,19 +228,6 @@ public class FlowInputOutput {
|
||||
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
|
||||
})
|
||||
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||
if (resolved.size() < data.size()) {
|
||||
RunContext runContext = runContextFactory.of(flow, execution);
|
||||
for (var inputKey : data.keySet()) {
|
||||
if (!resolved.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
|
||||
inputKey,
|
||||
flow.getNamespace(),
|
||||
flow.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return MapUtils.flattenToNestedMap(resolved);
|
||||
}
|
||||
|
||||
@@ -326,13 +321,13 @@ public class FlowInputOutput {
|
||||
resolvable.setInput(input);
|
||||
|
||||
Object value = resolvable.get().value();
|
||||
|
||||
|
||||
// resolve default if needed
|
||||
if (value == null && input.getDefaults() != null) {
|
||||
value = resolveDefaultValue(input, runContext);
|
||||
resolvable.isDefault(true);
|
||||
}
|
||||
|
||||
|
||||
// validate and parse input value
|
||||
if (value == null) {
|
||||
if (input.getRequired()) {
|
||||
@@ -361,7 +356,7 @@ public class FlowInputOutput {
|
||||
|
||||
return resolvable.get();
|
||||
}
|
||||
|
||||
|
||||
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
|
||||
return switch (input.getType()) {
|
||||
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
|
||||
@@ -378,7 +373,7 @@ public class FlowInputOutput {
|
||||
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
|
||||
@@ -464,7 +459,7 @@ public class FlowInputOutput {
|
||||
if (data.getType() == null) {
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
|
||||
}
|
||||
|
||||
|
||||
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
|
||||
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(
|
||||
@@ -541,17 +536,17 @@ public class FlowInputOutput {
|
||||
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
if (outputs == null) return Map.of();
|
||||
|
||||
|
||||
// render required outputs
|
||||
Map<String, Object> outputsById = outputs
|
||||
.stream()
|
||||
.filter(output -> output.getRequired() == null || output.getRequired())
|
||||
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
|
||||
outputsById = runContext.render(outputsById);
|
||||
|
||||
|
||||
// render optional outputs one by one to catch, log, and skip any error.
|
||||
for (io.kestra.core.models.flows.Output output : outputs) {
|
||||
if (Boolean.FALSE.equals(output.getRequired())) {
|
||||
@@ -594,9 +589,9 @@ public class FlowInputOutput {
|
||||
}
|
||||
|
||||
public void isDefault(boolean isDefault) {
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
}
|
||||
|
||||
|
||||
public void setInput(final Input<?> input) {
|
||||
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
|
||||
}
|
||||
|
||||
@@ -84,12 +84,6 @@ public class FlowableUtils {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have submitted, leave
|
||||
Optional<TaskRun> lastSubmitted = execution.findLastSubmitted(taskRuns);
|
||||
if (lastSubmitted.isPresent()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have running, leave
|
||||
Optional<TaskRun> lastRunning = execution.findLastRunning(taskRuns);
|
||||
if (lastRunning.isPresent()) {
|
||||
|
||||
@@ -168,7 +168,6 @@ public class Extension extends AbstractExtension {
|
||||
functions.put("randomPort", new RandomPortFunction());
|
||||
functions.put("fileExists", fileExistsFunction);
|
||||
functions.put("isFileEmpty", isFileEmptyFunction);
|
||||
functions.put("nanoId", new NanoIDFunction());
|
||||
functions.put("tasksWithState", new TasksWithStateFunction());
|
||||
functions.put(HttpFunction.NAME, httpFunction);
|
||||
return functions;
|
||||
|
||||
@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
|
||||
ZoneId zoneId = zoneId(timeZone);
|
||||
ZonedDateTime date = convert(input, zoneId, existingFormat);
|
||||
|
||||
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import io.kestra.core.http.HttpRequest;
|
||||
import io.kestra.core.http.HttpResponse;
|
||||
import io.kestra.core.http.client.HttpClient;
|
||||
import io.kestra.core.http.client.HttpClientException;
|
||||
import io.kestra.core.http.client.HttpClientRequestException;
|
||||
import io.kestra.core.http.client.HttpClientResponseException;
|
||||
import io.kestra.core.http.client.configurations.HttpConfiguration;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
@@ -103,15 +101,8 @@ public class HttpFunction<T> implements Function {
|
||||
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
|
||||
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
|
||||
return response.getBody();
|
||||
} catch (HttpClientResponseException e) {
|
||||
if (e.getResponse() != null) {
|
||||
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
|
||||
throw new PebbleException(e, msg , lineNumber, self.getName());
|
||||
} else {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
|
||||
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package io.kestra.core.runners.pebble.functions;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Function;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NanoIDFunction implements Function {
|
||||
|
||||
private static final int DEFAULT_LENGTH = 21;
|
||||
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
private static final String LENGTH = "length";
|
||||
private static final String ALPHABET = "alphabet";
|
||||
|
||||
private static final int MAX_LENGTH = 1000;
|
||||
|
||||
@Override
|
||||
public Object execute(
|
||||
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
|
||||
int length = DEFAULT_LENGTH;
|
||||
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
|
||||
length = parseLength(args, self, lineNumber);
|
||||
}
|
||||
char[] alphabet = DEFAULT_ALPHABET;
|
||||
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
|
||||
alphabet = ((String) args.get(ALPHABET)).toCharArray();
|
||||
}
|
||||
return createNanoID(length, alphabet);
|
||||
}
|
||||
|
||||
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
|
||||
var value = (Long) args.get(LENGTH);
|
||||
if(value > MAX_LENGTH) {
|
||||
throw new PebbleException(
|
||||
null,
|
||||
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
|
||||
lineNumber,
|
||||
self.getName());
|
||||
}
|
||||
return Math.toIntExact(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
return List.of(LENGTH,ALPHABET);
|
||||
}
|
||||
|
||||
String createNanoID(int length, char[] alphabet){
|
||||
final char[] data = new char[length];
|
||||
final byte[] bytes = new byte[length];
|
||||
final int mask = alphabet.length-1;
|
||||
secureRandom.nextBytes(bytes);
|
||||
for (int i = 0; i < length; ++i) {
|
||||
data[i] = alphabet[bytes[i] & mask];
|
||||
}
|
||||
return String.valueOf(data);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -180,13 +180,23 @@ public final class FileSerde {
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,8 +718,8 @@ public class ExecutionService {
|
||||
} else {
|
||||
newExecution = execution.withState(killingOrAfterKillState);
|
||||
}
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
// immediately without publishing a CrudEvent like it's done on pause/resume method.
|
||||
return newExecution;
|
||||
}
|
||||
|
||||
@@ -5,14 +5,11 @@ import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class SkipExecutionService {
|
||||
private volatile List<String> skipExecutions = Collections.emptyList();
|
||||
private volatile List<FlowId> skipFlows = Collections.emptyList();
|
||||
@@ -25,11 +22,11 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
public synchronized void setSkipFlows(List<String> skipFlows) {
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).filter(Objects::nonNull).toList();
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipNamespaces(List<String> skipNamespaces) {
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).filter(Objects::nonNull).toList();
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipTenants(List<String> skipTenants) {
|
||||
@@ -76,31 +73,19 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
record FlowId(String tenant, String namespace, String flow) {
|
||||
static @Nullable FlowId from(String flowId) {
|
||||
static FlowId from(String flowId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(flowId);
|
||||
if (parts.length == 3) {
|
||||
return new FlowId(parts[0], parts[1], parts[2]);
|
||||
} else if (parts.length == 2) {
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid flow skip with values: '{}'", flowId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
|
||||
record NamespaceId(String tenant, String namespace) {
|
||||
static @Nullable NamespaceId from(String namespaceId) {
|
||||
static NamespaceId from(String namespaceId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(namespaceId);
|
||||
|
||||
if (parts.length == 2) {
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid namespace skip with values:'{}'", namespaceId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,12 +14,8 @@ import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
@@ -36,11 +32,7 @@ public abstract class StorageService {
|
||||
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(runContext.storage().getFile(from)))) {
|
||||
List<Path> splited;
|
||||
|
||||
if (storageSplitInterface.getRegexPattern() != null) {
|
||||
String renderedPattern = runContext.render(storageSplitInterface.getRegexPattern()).as(String.class).orElseThrow();
|
||||
String separator = runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow();
|
||||
splited = StorageService.splitByRegex(runContext, extension, separator, bufferedReader, renderedPattern);
|
||||
} else if (storageSplitInterface.getBytes() != null) {
|
||||
if (storageSplitInterface.getBytes() != null) {
|
||||
ReadableBytesTypeConverter readableBytesTypeConverter = new ReadableBytesTypeConverter();
|
||||
Number convert = readableBytesTypeConverter.convert(runContext.render(storageSplitInterface.getBytes()).as(String.class).orElseThrow(), Number.class)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Invalid size with value '" + storageSplitInterface.getBytes() + "'"));
|
||||
@@ -55,7 +47,7 @@ public abstract class StorageService {
|
||||
splited = StorageService.split(runContext, extension, runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow(),
|
||||
bufferedReader, (bytes, size) -> size >= renderedRows);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, rows, nor regexPattern");
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, nor rows");
|
||||
}
|
||||
|
||||
return splited
|
||||
@@ -125,36 +117,4 @@ public abstract class StorageService {
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
private static List<Path> splitByRegex(RunContext runContext, String extension, String separator, BufferedReader bufferedReader, String regexPattern) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
Map<String, RandomAccessFile> writers = new HashMap<>();
|
||||
Pattern pattern = Pattern.compile(regexPattern);
|
||||
|
||||
String row;
|
||||
while ((row = bufferedReader.readLine()) != null) {
|
||||
Matcher matcher = pattern.matcher(row);
|
||||
|
||||
if (matcher.find() && matcher.groupCount() > 0) {
|
||||
String routingKey = matcher.group(1);
|
||||
|
||||
// Get or create writer for this routing key
|
||||
RandomAccessFile writer = writers.get(routingKey);
|
||||
if (writer == null) {
|
||||
Path path = runContext.workingDir().createTempFile(extension);
|
||||
files.add(path);
|
||||
writer = new RandomAccessFile(path.toFile(), "rw");
|
||||
writers.put(routingKey, writer);
|
||||
}
|
||||
|
||||
byte[] bytes = (row + separator).getBytes(StandardCharsets.UTF_8);
|
||||
writer.getChannel().write(ByteBuffer.wrap(bytes));
|
||||
}
|
||||
// Lines that don't match the pattern are ignored
|
||||
}
|
||||
|
||||
writers.values().forEach(throwConsumer(RandomAccessFile::close));
|
||||
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -26,13 +26,4 @@ public interface StorageSplitInterface {
|
||||
defaultValue = "\\n"
|
||||
)
|
||||
Property<String> getSeparator();
|
||||
|
||||
@Schema(
|
||||
title = "Split file by regex pattern with the first capture group as the routing key.",
|
||||
description = "A regular expression pattern with a capture group. Lines matching this pattern will be grouped by the captured value. " +
|
||||
"For example, `\\[(\\w+)\\]` will group lines by log level (ERROR, WARN, INFO) extracted from log entries. " +
|
||||
"Lines with the same captured value will be written to the same output file. " +
|
||||
"This allows content-based splitting where the file is divided based on data patterns rather than size or line count."
|
||||
)
|
||||
Property<String> getRegexPattern();
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@ abstract public class TruthUtils {
|
||||
private static final List<String> FALSE_VALUES = List.of("false", "0", "-0", "");
|
||||
|
||||
public static boolean isTruthy(String condition) {
|
||||
return condition != null && !FALSE_VALUES.contains(condition.trim());
|
||||
return condition != null && !FALSE_VALUES.contains(condition);
|
||||
}
|
||||
|
||||
public static boolean isFalsy(String condition) {
|
||||
return condition != null && FALSE_VALUES.contains(condition.trim());
|
||||
return condition != null && FALSE_VALUES.contains(condition);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,84 +32,48 @@ public class Version implements Comparable<Version> {
|
||||
* @param version the version.
|
||||
* @return a new {@link Version} instance.
|
||||
*/
|
||||
public static Version of(final Object version) {
|
||||
public static Version of(String version) {
|
||||
|
||||
if (Objects.isNull(version)) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse null version");
|
||||
}
|
||||
|
||||
String strVersion = version.toString();
|
||||
|
||||
if (strVersion.startsWith("v")) {
|
||||
strVersion = strVersion.substring(1);
|
||||
if (version.startsWith("v")) {
|
||||
version = version.substring(1);
|
||||
}
|
||||
|
||||
int qualifier = strVersion.indexOf("-");
|
||||
int qualifier = version.indexOf("-");
|
||||
|
||||
final String[] versions = qualifier > 0 ?
|
||||
strVersion.substring(0, qualifier).split("\\.") :
|
||||
strVersion.split("\\.");
|
||||
version.substring(0, qualifier).split("\\.") :
|
||||
version.split("\\.");
|
||||
try {
|
||||
final int majorVersion = Integer.parseInt(versions[0]);
|
||||
final Integer minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : null;
|
||||
final Integer incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : null;
|
||||
final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0;
|
||||
final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0;
|
||||
|
||||
return new Version(
|
||||
majorVersion,
|
||||
minorVersion,
|
||||
incrementalVersion,
|
||||
qualifier > 0 ? strVersion.substring(qualifier + 1) : null,
|
||||
strVersion
|
||||
qualifier > 0 ? version.substring(qualifier + 1) : null,
|
||||
version
|
||||
);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resolves the most appropriate stable version from a collection, based on a given input version.
|
||||
* <p>
|
||||
* The matching rules are:
|
||||
* <ul>
|
||||
* <li>If {@code from} specifies only a major version (e.g. {@code 1}), return the latest stable version
|
||||
* with the same major (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a major and minor version only (e.g. {@code 1.2}), return the latest
|
||||
* stable version with the same major and minor (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a full version with major, minor, and patch (e.g. {@code 1.2.2}),
|
||||
* then only return it if it is exactly present (and stable) in {@code versions}.
|
||||
* No "upgrade" is performed in this case.</li>
|
||||
* <li>If no suitable version is found, returns {@code null}.</li>
|
||||
* </ul>
|
||||
* Static helper method for returning the most recent stable version for a current {@link Version}.
|
||||
*
|
||||
* @param from the reference version (may specify only major, or major+minor, or major+minor+patch).
|
||||
* @param versions the collection of candidate versions to resolve against.
|
||||
* @return the best matching stable version, or {@code null} if none match.
|
||||
* @param from the current version.
|
||||
* @param versions the list of version.
|
||||
*
|
||||
* @return the last stable version.
|
||||
*/
|
||||
public static Version getStable(final Version from, final Collection<Version> versions) {
|
||||
// Case 1: "from" is only a major (e.g. 1)
|
||||
if (from.hasOnlyMajor()) {
|
||||
List<Version> sameMajor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion())
|
||||
.toList();
|
||||
return sameMajor.isEmpty() ? null : Version.getLatest(sameMajor);
|
||||
}
|
||||
|
||||
// Case 2: "from" is major+minor only (e.g. 1.2)
|
||||
if (from.hasMajorAndMinorOnly()) {
|
||||
List<Version> sameMinor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion()
|
||||
&& v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
return sameMinor.isEmpty() ? null : Version.getLatest(sameMinor);
|
||||
}
|
||||
|
||||
// Case 3: "from" is full version (major+minor+patch)
|
||||
if (versions.contains(from)) {
|
||||
return from;
|
||||
}
|
||||
|
||||
// No match
|
||||
return null;
|
||||
List<Version> compatibleVersions = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
if (compatibleVersions.isEmpty()) return null;
|
||||
return Version.getLatest(compatibleVersions);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -159,8 +123,8 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
private final int majorVersion;
|
||||
private final Integer minorVersion;
|
||||
private final Integer patchVersion;
|
||||
private final int minorVersion;
|
||||
private final int incrementalVersion;
|
||||
private final Qualifier qualifier;
|
||||
|
||||
private final String originalVersion;
|
||||
@@ -170,14 +134,14 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
*/
|
||||
public Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int patchVersion,
|
||||
final int incrementalVersion,
|
||||
final String qualifier) {
|
||||
this(majorVersion, minorVersion, patchVersion, qualifier, null);
|
||||
this(majorVersion, minorVersion, incrementalVersion, qualifier, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -185,25 +149,25 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
* @param originalVersion the original string version.
|
||||
*/
|
||||
private Version(final Integer majorVersion,
|
||||
final Integer minorVersion,
|
||||
final Integer patchVersion,
|
||||
private Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
final String qualifier,
|
||||
final String originalVersion) {
|
||||
this.majorVersion = requirePositive(majorVersion, "major");
|
||||
this.minorVersion = requirePositive(minorVersion, "minor");
|
||||
this.patchVersion = requirePositive(patchVersion, "incremental");
|
||||
this.incrementalVersion = requirePositive(incrementalVersion, "incremental");
|
||||
this.qualifier = qualifier != null ? new Qualifier(qualifier) : null;
|
||||
this.originalVersion = originalVersion;
|
||||
}
|
||||
|
||||
|
||||
private static Integer requirePositive(Integer version, final String message) {
|
||||
if (version != null && version < 0) {
|
||||
private static int requirePositive(int version, final String message) {
|
||||
if (version < 0) {
|
||||
throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message));
|
||||
}
|
||||
return version;
|
||||
@@ -214,11 +178,11 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
public int minorVersion() {
|
||||
return minorVersion != null ? minorVersion : 0;
|
||||
return minorVersion;
|
||||
}
|
||||
|
||||
public int patchVersion() {
|
||||
return patchVersion != null ? patchVersion : 0;
|
||||
public int incrementalVersion() {
|
||||
return incrementalVersion;
|
||||
}
|
||||
|
||||
public Qualifier qualifier() {
|
||||
@@ -233,9 +197,9 @@ public class Version implements Comparable<Version> {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof Version)) return false;
|
||||
Version version = (Version) o;
|
||||
return Objects.equals(majorVersion,version.majorVersion) &&
|
||||
Objects.equals(minorVersion, version.minorVersion) &&
|
||||
Objects.equals(patchVersion,version.patchVersion) &&
|
||||
return majorVersion == version.majorVersion &&
|
||||
minorVersion == version.minorVersion &&
|
||||
incrementalVersion == version.incrementalVersion &&
|
||||
Objects.equals(qualifier, version.qualifier);
|
||||
}
|
||||
|
||||
@@ -244,7 +208,7 @@ public class Version implements Comparable<Version> {
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(majorVersion, minorVersion, patchVersion, qualifier);
|
||||
return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -254,7 +218,7 @@ public class Version implements Comparable<Version> {
|
||||
public String toString() {
|
||||
if (originalVersion != null) return originalVersion;
|
||||
|
||||
String version = majorVersion + "." + minorVersion + "." + patchVersion;
|
||||
String version = majorVersion + "." + minorVersion + "." + incrementalVersion;
|
||||
return (qualifier != null) ? version +"-" + qualifier : version;
|
||||
}
|
||||
|
||||
@@ -274,7 +238,7 @@ public class Version implements Comparable<Version> {
|
||||
return compareMinor;
|
||||
}
|
||||
|
||||
int compareIncremental = Integer.compare(that.patchVersion, this.patchVersion);
|
||||
int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion);
|
||||
if (compareIncremental != 0) {
|
||||
return compareIncremental;
|
||||
}
|
||||
@@ -289,21 +253,6 @@ public class Version implements Comparable<Version> {
|
||||
|
||||
return this.qualifier.compareTo(that.qualifier);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true if only major is specified (e.g. "1")
|
||||
*/
|
||||
private boolean hasOnlyMajor() {
|
||||
return minorVersion == null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if major+minor are specified, but no patch (e.g. "1.2")
|
||||
*/
|
||||
private boolean hasMajorAndMinorOnly() {
|
||||
return minorVersion != null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether this version is before the given one.
|
||||
|
||||
@@ -46,19 +46,16 @@ public class VersionProvider {
|
||||
this.date = loadTime(gitProperties);
|
||||
this.version = loadVersion(buildProperties, gitProperties);
|
||||
|
||||
// check the version in the settings and update if needed, we didn't use it would allow us to detect incompatible update later if needed
|
||||
settingRepository.ifPresent(
|
||||
settingRepositoryInterface -> persistVersion(settingRepositoryInterface, version));
|
||||
}
|
||||
|
||||
private static synchronized void persistVersion(SettingRepositoryInterface settingRepositoryInterface, String version) {
|
||||
Optional<Setting> versionSetting = settingRepositoryInterface.findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(version)) {
|
||||
settingRepositoryInterface.save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(version)
|
||||
.build()
|
||||
);
|
||||
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
|
||||
if (settingRepository.isPresent()) {
|
||||
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
|
||||
settingRepository.get().save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(this.version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilter;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display Flow data in a dashboard chart."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display a chart with a list of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: list_flows
|
||||
type: io.kestra.plugin.core.dashboard.chart.Table
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.Flows
|
||||
columns:
|
||||
namespace:
|
||||
field: NAMESPACE
|
||||
id:
|
||||
field: ID
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("Flows")
|
||||
public class Flows<C extends ColumnDescriptor<Flows.Fields>> extends DataFilter<Flows.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilterKPI;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display a chart with Flows KPI.",
|
||||
description = "Change."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display count of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: kpi
|
||||
type: io.kestra.plugin.core.dashboard.chart.KPI
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.FlowsKPI
|
||||
columns:
|
||||
field: ID
|
||||
agg: COUNT
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("FlowsKPI")
|
||||
public class FlowsKPI<C extends ColumnDescriptor<FlowsKPI.Fields>> extends DataFilterKPI<FlowsKPI.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.dashboards.filters.AbstractFilter;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public interface IFlows extends IData<IFlows.Fields> {
|
||||
|
||||
default List<AbstractFilter<IFlows.Fields>> whereWithGlobalFilters(List<QueryFilter> filters, ZonedDateTime startDate, ZonedDateTime endDate, List<AbstractFilter<IFlows.Fields>> where) {
|
||||
List<AbstractFilter<IFlows.Fields>> updatedWhere = where != null ? new ArrayList<>(where) : new ArrayList<>();
|
||||
|
||||
if (ListUtils.isEmpty(filters)) {
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
List<QueryFilter> namespaceFilters = filters.stream().filter(f -> f.field().equals(QueryFilter.Field.NAMESPACE)).toList();
|
||||
if (!namespaceFilters.isEmpty()) {
|
||||
updatedWhere.removeIf(filter -> filter.getField().equals(IFlows.Fields.NAMESPACE));
|
||||
namespaceFilters.forEach(f -> {
|
||||
updatedWhere.add(f.toDashboardFilterBuilder(IFlows.Fields.NAMESPACE, f.value()));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
enum Fields {
|
||||
ID,
|
||||
NAMESPACE,
|
||||
REVISION
|
||||
}
|
||||
}
|
||||
@@ -127,24 +127,9 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
}
|
||||
|
||||
// check for system labels: none can be passed at runtime
|
||||
Optional<Map.Entry<String, String>> systemLabel = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX))
|
||||
.findFirst();
|
||||
if (systemLabel.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"System labels can only be set by Kestra itself, offending label: " +
|
||||
systemLabel.get().getKey() + "=" + systemLabel.get().getValue()
|
||||
);
|
||||
}
|
||||
|
||||
// check for empty label values
|
||||
Optional<Map.Entry<String, String>> emptyValue = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().isEmpty())
|
||||
.findFirst();
|
||||
if (emptyValue.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label values cannot be empty, offending label: " + emptyValue.get().getKey()
|
||||
);
|
||||
Optional<Map.Entry<String, String>> first = labelsAsMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX)).findFirst();
|
||||
if (first.isPresent()) {
|
||||
throw new IllegalArgumentException("System labels can only be set by Kestra itself, offending label: " + first.get().getKey() + "=" + first.get().getValue());
|
||||
}
|
||||
|
||||
Map<String, String> newLabels = ListUtils.emptyOnNull(execution.getLabels()).stream()
|
||||
@@ -155,7 +140,6 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
newLabels.putAll(labelsAsMap);
|
||||
|
||||
return execution.withLabels(newLabels.entrySet().stream()
|
||||
.filter(Label.getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(
|
||||
entry.getKey(),
|
||||
entry.getValue()
|
||||
|
||||
@@ -478,7 +478,7 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
try (InputStream is = runContext.storage().getFile(splitsURI)){
|
||||
String fileContent = new String(is.readAllBytes());
|
||||
List<URI> splits = fileContent.lines().map(line -> URI.create(line)).toList();
|
||||
AtomicInteger currentIteration = new AtomicInteger(0);
|
||||
AtomicInteger currentIteration = new AtomicInteger(1);
|
||||
|
||||
return splits
|
||||
.stream()
|
||||
@@ -648,8 +648,6 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
|
||||
@Builder.Default
|
||||
private Property<String> separator = Property.ofValue("\n");
|
||||
|
||||
private Property<String> regexPattern;
|
||||
}
|
||||
|
||||
@Builder
|
||||
|
||||
@@ -173,8 +173,8 @@ public class Download extends AbstractHttp implements RunnableTask<Download.Outp
|
||||
if (path.indexOf('/') != -1) {
|
||||
path = path.substring(path.lastIndexOf('/')); // keep the last segment
|
||||
}
|
||||
if (path.lastIndexOf('.') != -1) {
|
||||
return path.substring(path.lastIndexOf('.'));
|
||||
if (path.indexOf('.') != -1) {
|
||||
return path.substring(path.indexOf('.'));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -48,13 +48,6 @@ import java.util.List;
|
||||
"partitions: 8"
|
||||
}
|
||||
),
|
||||
@Example(
|
||||
title = "Split a file by regex pattern - group lines by captured value.",
|
||||
code = {
|
||||
"from: \"kestra://long/url/logs.txt\"",
|
||||
"regexPattern: \"\\\\[(\\\\w+)\\\\]\""
|
||||
}
|
||||
),
|
||||
},
|
||||
aliases = "io.kestra.core.tasks.storages.Split"
|
||||
)
|
||||
@@ -72,13 +65,6 @@ public class Split extends Task implements RunnableTask<Split.Output>, StorageSp
|
||||
|
||||
private Property<Integer> rows;
|
||||
|
||||
@Schema(
|
||||
title = "Split file by regex pattern. Lines are grouped by the first capture group value.",
|
||||
description = "A regular expression pattern with a capture group. Lines matching this pattern will be grouped by the captured value. For example, `\\[(\\w+)\\]` will group lines by log level (ERROR, WARN, INFO) extracted from log entries."
|
||||
)
|
||||
@PluginProperty(dynamic = true)
|
||||
private Property<String> regexPattern;
|
||||
|
||||
@Builder.Default
|
||||
private Property<String> separator = Property.ofValue("\n");
|
||||
|
||||
|
||||
@@ -48,38 +48,6 @@ import jakarta.validation.constraints.Size;
|
||||
- 200 if the webhook triggers an execution.
|
||||
- 204 if the webhook cannot trigger an execution due to a lack of matching event conditions sent by other application.
|
||||
|
||||
The response body will contain the execution ID if the execution is successfully triggered using the following format:
|
||||
```json
|
||||
{
|
||||
"tenantId": "your_tenant_id",
|
||||
"namespace": "your_namespace",
|
||||
"flowId": "your_flow_id",
|
||||
"flowRevision": 1,
|
||||
"trigger": {
|
||||
"id": "the_trigger_id",
|
||||
"type": "io.kestra.plugin.core.trigger.Webhook",
|
||||
"variables": {
|
||||
# The variables sent by the webhook caller
|
||||
},
|
||||
"logFile": "the_log_file_url"
|
||||
},
|
||||
"outputs": {
|
||||
# The outputs of the flow, only available if `wait` is set to true
|
||||
},
|
||||
"labels": [
|
||||
{"key": "value" }
|
||||
],
|
||||
"state": {
|
||||
"type": "RUNNING",
|
||||
"histories": [
|
||||
# The state histories of the execution
|
||||
]
|
||||
},
|
||||
"url": "the_execution_url_inside_ui",
|
||||
}
|
||||
```
|
||||
If you set the `wait` property to `true` and `returnOutputs` to `true`, the webhook call will wait for the flow to finish and return the flow outputs as response.
|
||||
|
||||
A webhook trigger can have conditions, but it doesn't support conditions of type `MultipleCondition`."""
|
||||
)
|
||||
@Plugin(
|
||||
@@ -148,23 +116,8 @@ public class Webhook extends AbstractTrigger implements TriggerOutput<Webhook.Ou
|
||||
|
||||
@PluginProperty
|
||||
@Builder.Default
|
||||
@Schema(
|
||||
title = "Wait for the flow to finish.",
|
||||
description = """
|
||||
If set to `true` the webhook call will wait for the flow to finish and return the flow outputs as response.
|
||||
If set to `false` the webhook call will return immediately after the execution is created.
|
||||
"""
|
||||
)
|
||||
private Boolean wait = false;
|
||||
|
||||
@PluginProperty
|
||||
@Builder.Default
|
||||
@Schema(
|
||||
title = "Send outputs of the flows as response for webhook caller.",
|
||||
description = "Requires `wait` to be `true`."
|
||||
)
|
||||
private Boolean returnOutputs = false;
|
||||
|
||||
public Optional<Execution> evaluate(HttpRequest<String> request, io.kestra.core.models.flows.Flow flow) {
|
||||
String body = request.getBody().orElse(null);
|
||||
|
||||
|
||||
@@ -21,13 +21,10 @@ import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class DocumentationGeneratorTest {
|
||||
@Inject
|
||||
JsonSchemaGenerator jsonSchemaGenerator;
|
||||
|
||||
@@ -37,7 +37,6 @@ import lombok.Value;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
@@ -68,7 +67,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@Testcontainers
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
class HttpClientTest {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
@@ -1,32 +1,19 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.validations.ModelValidator;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class LabelTest {
|
||||
@Inject
|
||||
private ModelValidator modelValidator;
|
||||
|
||||
@Test
|
||||
void shouldGetNestedMapGivenDistinctLabels() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label(Label.USERNAME, "test"),
|
||||
new Label(Label.CORRELATION_ID, "id"),
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null)
|
||||
)
|
||||
new Label(Label.CORRELATION_ID, "id"))
|
||||
);
|
||||
|
||||
assertThat(result).isEqualTo(
|
||||
@@ -47,18 +34,6 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toNestedMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, Object> result = Label.toNestedMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGetMapGivenDistinctLabels() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
@@ -84,18 +59,6 @@ class LabelTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toMapShouldIgnoreEmptyOrNull() {
|
||||
Map<String, String> result = Label.toMap(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDuplicateLabelsWithKeyOrderKept() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
@@ -110,28 +73,4 @@ class LabelTest {
|
||||
new Label(Label.CORRELATION_ID, "id")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deduplicateShouldIgnoreEmptyAndNull() {
|
||||
List<Label> result = Label.deduplicate(List.of(
|
||||
new Label("", "bar"),
|
||||
new Label(null, "bar"),
|
||||
new Label("foo", ""),
|
||||
new Label("baz", null))
|
||||
);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldValidateEmpty() {
|
||||
Optional<ConstraintViolationException> validLabelResult = modelValidator.isValid(new Label("foo", "bar"));
|
||||
assertThat(validLabelResult.isPresent()).isFalse();
|
||||
|
||||
Optional<ConstraintViolationException> emptyValueLabelResult = modelValidator.isValid(new Label("foo", ""));
|
||||
assertThat(emptyValueLabelResult.isPresent()).isTrue();
|
||||
|
||||
Optional<ConstraintViolationException> emptyKeyLabelResult = modelValidator.isValid(new Label("", "bar"));
|
||||
assertThat(emptyKeyLabelResult.isPresent()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -13,19 +13,19 @@ import java.util.Map;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ExecutionTest {
|
||||
private static final TaskRun.TaskRunBuilder TASK_RUN = TaskRun.builder()
|
||||
.id("test");
|
||||
|
||||
@Test
|
||||
void hasTaskRunJoinableTrue() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -36,15 +36,13 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableSameState() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State())
|
||||
.build())
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State())
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -53,8 +51,7 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableFailedExecutionFromExecutor() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -62,8 +59,7 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State(State.Type.RUNNING, new State()))
|
||||
.build()
|
||||
)).isFalse();
|
||||
@@ -72,8 +68,7 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartFailed() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -82,8 +77,7 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State(State.Type.FAILED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
))
|
||||
@@ -94,8 +88,7 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableRestartSuccess() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -104,8 +97,7 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.SUCCESS)
|
||||
@@ -117,8 +109,7 @@ class ExecutionTest {
|
||||
@Test
|
||||
void hasTaskRunJoinableAfterRestart() {
|
||||
Execution execution = Execution.builder()
|
||||
.taskRunList(Collections.singletonList(TaskRun.builder()
|
||||
.id("test")
|
||||
.taskRunList(Collections.singletonList(TASK_RUN
|
||||
.state(new State(State.Type.CREATED, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
@@ -127,8 +118,7 @@ class ExecutionTest {
|
||||
)
|
||||
.build();
|
||||
|
||||
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
|
||||
.id("test")
|
||||
assertThat(execution.hasTaskRunJoinable(TASK_RUN
|
||||
.state(new State(State.Type.SUCCESS, new State()
|
||||
.withState(State.Type.RUNNING)
|
||||
.withState(State.Type.FAILED)
|
||||
|
||||
@@ -7,11 +7,12 @@ import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.TriggerRepositoryInterface;
|
||||
import io.kestra.core.runners.TestRunnerUtils;
|
||||
import io.kestra.core.runners.RunnerUtils;
|
||||
import io.kestra.core.serializers.YamlParser;
|
||||
import io.kestra.core.services.GraphService;
|
||||
import io.kestra.core.utils.GraphUtils;
|
||||
@@ -44,7 +45,7 @@ class FlowGraphTest {
|
||||
private TriggerRepositoryInterface triggerRepositoryInterface;
|
||||
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
@Test
|
||||
void simple() throws IllegalVariableEvaluationException, IOException {
|
||||
@@ -260,10 +261,10 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant1")
|
||||
@LoadFlows({"flows/valids/task-flow.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
void subflow() throws IllegalVariableEvaluationException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml", "tenant1");
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml");
|
||||
FlowGraph flowGraph = GraphUtils.flowGraph(flow, null);
|
||||
|
||||
assertThat(flowGraph.getNodes().size()).isEqualTo(6);
|
||||
@@ -292,15 +293,15 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"}, tenantId = "tenant2")
|
||||
@LoadFlows({"flows/valids/task-flow-dynamic.yaml",
|
||||
"flows/valids/switch.yaml"})
|
||||
void dynamicIdSubflow() throws IllegalVariableEvaluationException, TimeoutException, QueueException, IOException, FlowProcessingException {
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml", "tenant2").toBuilder().revision(1).build();
|
||||
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml").toBuilder().revision(1).build();
|
||||
|
||||
IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> graphService.flowGraph(flow, Collections.singletonList("root.launch")));
|
||||
assertThat(illegalArgumentException.getMessage()).isEqualTo("Can't expand subflow task 'launch' because namespace and/or flowId contains dynamic values. This can only be viewed on an execution.");
|
||||
|
||||
Execution execution = runnerUtils.runOne("tenant2", "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
|
||||
"namespace", f.getNamespace(),
|
||||
"flowId", "switch"
|
||||
));
|
||||
@@ -372,17 +373,13 @@ class FlowGraphTest {
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path) throws IOException {
|
||||
return parse(path, MAIN_TENANT);
|
||||
}
|
||||
|
||||
private FlowWithSource parse(String path, String tenantId) throws IOException {
|
||||
URL resource = TestsUtils.class.getClassLoader().getResource(path);
|
||||
assert resource != null;
|
||||
|
||||
File file = new File(resource.getFile());
|
||||
|
||||
return YamlParser.parse(file, FlowWithSource.class).toBuilder()
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.source(Files.readString(file.toPath()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -37,14 +37,14 @@ class PropertyTest {
|
||||
@Test
|
||||
void test() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.number(Property.ofExpression("{{numberValue}}"))
|
||||
.string(Property.ofExpression("{{stringValue}}"))
|
||||
.level(Property.ofExpression("{{levelValue}}"))
|
||||
.someDuration(Property.ofExpression("{{durationValue}}"))
|
||||
.withDefault(Property.ofExpression("{{defaultValue}}"))
|
||||
.items(Property.ofExpression("""
|
||||
.number(new Property<>("{{numberValue}}"))
|
||||
.string(new Property<>("{{stringValue}}"))
|
||||
.level(new Property<>("{{levelValue}}"))
|
||||
.someDuration(new Property<>("{{durationValue}}"))
|
||||
.withDefault(new Property<>("{{defaultValue}}"))
|
||||
.items(new Property<>("""
|
||||
["{{item1}}", "{{item2}}"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -87,13 +87,13 @@ class PropertyTest {
|
||||
@Test
|
||||
void withDefaultsAndMessagesFromList() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.number(Property.ofExpression("{{numberValue}}"))
|
||||
.string(Property.ofExpression("{{stringValue}}"))
|
||||
.level(Property.ofExpression("{{levelValue}}"))
|
||||
.someDuration(Property.ofExpression("{{durationValue}}"))
|
||||
.items(Property.ofExpression("""
|
||||
.number(new Property<>("{{numberValue}}"))
|
||||
.string(new Property<>("{{stringValue}}"))
|
||||
.level(new Property<>("{{levelValue}}"))
|
||||
.someDuration(new Property<>("{{durationValue}}"))
|
||||
.items(new Property<>("""
|
||||
["{{item1}}", "{{item2}}"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -156,14 +156,14 @@ class PropertyTest {
|
||||
}
|
||||
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.number(Property.ofExpression("{{numberValue}}"))
|
||||
.string(Property.ofExpression("{{stringValue}}"))
|
||||
.level(Property.ofExpression("{{levelValue}}"))
|
||||
.someDuration(Property.ofExpression("{{durationValue}}"))
|
||||
.withDefault(Property.ofExpression("{{defaultValue}}"))
|
||||
.items(Property.ofExpression("""
|
||||
.number(new Property<>("{{numberValue}}"))
|
||||
.string(new Property<>("{{stringValue}}"))
|
||||
.level(new Property<>("{{levelValue}}"))
|
||||
.someDuration(new Property<>("{{durationValue}}"))
|
||||
.withDefault(new Property<>("{{defaultValue}}"))
|
||||
.items(new Property<>("""
|
||||
["{{item1}}", "{{item2}}"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -202,12 +202,12 @@ class PropertyTest {
|
||||
@Test
|
||||
void failingToRender() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.number(Property.ofExpression("{{numberValue}}"))
|
||||
.string(Property.ofExpression("{{stringValue}}"))
|
||||
.level(Property.ofExpression("{{levelValue}}"))
|
||||
.someDuration(Property.ofExpression("{{durationValue}}"))
|
||||
.withDefault(Property.ofExpression("{{defaultValue}}"))
|
||||
.items(Property.ofExpression("""
|
||||
.number(new Property<>("{{numberValue}}"))
|
||||
.string(new Property<>("{{stringValue}}"))
|
||||
.level(new Property<>("{{levelValue}}"))
|
||||
.someDuration(new Property<>("{{durationValue}}"))
|
||||
.withDefault(new Property<>("{{defaultValue}}"))
|
||||
.items(new Property<>("""
|
||||
["{{item1}}", "{{item2}}"]"""))
|
||||
.from(Map.of("key", "{{mapValue}}"))
|
||||
.build();
|
||||
@@ -221,13 +221,13 @@ class PropertyTest {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.id("dynamic")
|
||||
.type(DynamicPropertyExampleTask.class.getName())
|
||||
.number(Property.ofExpression("{{numberValue}}"))
|
||||
.string(Property.ofExpression("{{stringValue}}"))
|
||||
.level(Property.ofExpression("{{levelValue}}"))
|
||||
.someDuration(Property.ofExpression("{{durationValue}}"))
|
||||
.items(Property.ofExpression("""
|
||||
.number(new Property<>("{{numberValue}}"))
|
||||
.string(new Property<>("{{stringValue}}"))
|
||||
.level(new Property<>("{{levelValue}}"))
|
||||
.someDuration(new Property<>("{{durationValue}}"))
|
||||
.items(new Property<>("""
|
||||
["{{item1}}", "{{item2}}"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -261,8 +261,8 @@ class PropertyTest {
|
||||
@Test
|
||||
void arrayAndMapToRender() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.items(Property.ofExpression("{{renderOnce(listToRender)}}"))
|
||||
.properties(Property.ofExpression("{{renderOnce(mapToRender)}}"))
|
||||
.items(new Property<>("{{renderOnce(listToRender)}}"))
|
||||
.properties(new Property<>("{{renderOnce(mapToRender)}}"))
|
||||
.build();
|
||||
var runContext = runContextFactory.of(Map.ofEntries(
|
||||
entry("arrayValueToRender", "arrayValue1"),
|
||||
@@ -284,9 +284,9 @@ class PropertyTest {
|
||||
@Test
|
||||
void aListToRender() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.items(Property.ofExpression("""
|
||||
.items(new Property<>("""
|
||||
["python test.py --input1 \\"{{ item1 }}\\" --input2 \\"{{ item2 }}\\"", "'gs://{{ renderOnce(\\"bucket\\") }}/{{ 'table' }}/{{ 'file' }}_*.csv.gz'"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -308,9 +308,9 @@ class PropertyTest {
|
||||
@Test
|
||||
void fromMessage() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.items(Property.ofExpression("""
|
||||
.items(new Property<>("""
|
||||
["python test.py --input1 \\"{{ item1 }}\\" --input2 \\"{{ item2 }}\\"", "'gs://{{ renderOnce(\\"bucket\\") }}/{{ 'table' }}/{{ 'file' }}_*.csv.gz'"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
@@ -335,9 +335,9 @@ class PropertyTest {
|
||||
@Test
|
||||
void fromListOfMessages() throws Exception {
|
||||
var task = DynamicPropertyExampleTask.builder()
|
||||
.items(Property.ofExpression("""
|
||||
.items(new Property<>("""
|
||||
["python test.py --input1 \\"{{ item1 }}\\" --input2 \\"{{ item2 }}\\"", "'gs://{{ renderOnce(\\"bucket\\") }}/{{ 'table' }}/{{ 'file' }}_*.csv.gz'"]"""))
|
||||
.properties(Property.ofExpression("""
|
||||
.properties(new Property<>("""
|
||||
{
|
||||
"key1": "{{value1}}",
|
||||
"key2": "{{value2}}"
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.runners.*;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mockito;
|
||||
@@ -85,9 +84,8 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfile() throws IOException {
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", namespace)));
|
||||
URI uri = createNsFile(false);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "namespace")));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
String fetchedContent = new String(fetch.readAllBytes());
|
||||
@@ -97,8 +95,7 @@ class URIFetcherTest {
|
||||
|
||||
@Test
|
||||
void shouldFetchFromNsfileFromOtherNs() throws IOException {
|
||||
String namespace = IdUtils.create();
|
||||
URI uri = createNsFile(namespace, true);
|
||||
URI uri = createNsFile(true);
|
||||
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "other")));
|
||||
|
||||
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
|
||||
@@ -142,7 +139,8 @@ class URIFetcherTest {
|
||||
);
|
||||
}
|
||||
|
||||
private URI createNsFile(String namespace, boolean nsInAuthority) throws IOException {
|
||||
private URI createNsFile(boolean nsInAuthority) throws IOException {
|
||||
String namespace = "namespace";
|
||||
String filePath = "file.txt";
|
||||
storage.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storage.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
|
||||
@@ -10,7 +10,6 @@ import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -34,15 +33,17 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorage() throws IOException {
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
var runContext = runContextFactory.of();
|
||||
var command = ScriptService.replaceInternalStorage(runContext, null, false);
|
||||
assertThat(command).isEqualTo("");
|
||||
|
||||
command = ScriptService.replaceInternalStorage(runContext, "my command", false);
|
||||
assertThat(command).isEqualTo("my command");
|
||||
|
||||
Path path = createFile(tenant, "file");
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
File localFile = null;
|
||||
@@ -69,10 +70,12 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void replaceInternalStorageUnicode() throws IOException {
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
var runContext = runContextFactory.of();
|
||||
|
||||
Path path = createFile(tenant, "file-龍");
|
||||
Path path = Path.of("/tmp/unittest/main/file-龍.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
|
||||
String internalStorageUri = "kestra://some/file-龍.txt";
|
||||
File localFile = null;
|
||||
@@ -92,10 +95,12 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadInputFiles() throws IOException {
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
var runContext = runContextFactory.of();
|
||||
|
||||
Path path = createFile(tenant, "file");
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
|
||||
List<File> filesToDelete = new ArrayList<>();
|
||||
String internalStorageUri = "kestra://some/file.txt";
|
||||
@@ -138,11 +143,13 @@ class ScriptServiceTest {
|
||||
|
||||
@Test
|
||||
void uploadOutputFiles() throws IOException {
|
||||
String tenant = IdUtils.create();
|
||||
var runContext = runContextFactory.of("id", "namespace", tenant);
|
||||
Path path = createFile(tenant, "file");
|
||||
var runContext = runContextFactory.of();
|
||||
Path path = Path.of("/tmp/unittest/main/file.txt");
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createFile(path);
|
||||
}
|
||||
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/main"));
|
||||
assertThat(outputFiles, not(anEmptyMap()));
|
||||
assertThat(outputFiles.get("file.txt")).isEqualTo(URI.create("kestra:///file.txt"));
|
||||
|
||||
@@ -225,13 +232,4 @@ class ScriptServiceTest {
|
||||
.build();
|
||||
return runContextFactory.of(flow, task, execution, taskRun);
|
||||
}
|
||||
|
||||
private static Path createFile(String tenant, String fileName) throws IOException {
|
||||
Path path = Path.of("/tmp/unittest/%s/%s.txt".formatted(tenant, fileName));
|
||||
if (!path.toFile().exists()) {
|
||||
Files.createDirectory(Path.of("/tmp/unittest/%s".formatted(tenant)));
|
||||
Files.createFile(path);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package io.kestra.core.models.triggers.multipleflows;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import io.kestra.plugin.core.condition.ExecutionFlow;
|
||||
@@ -34,9 +33,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void allDefault() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -52,9 +50,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void daily() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -70,9 +67,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void dailyAdvance() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -88,9 +84,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void hourly() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -107,9 +102,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void minutely() {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
|
||||
@@ -121,9 +115,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expiration() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -143,9 +136,8 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
@Test
|
||||
void expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -154,21 +146,20 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -177,21 +168,20 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isZero();
|
||||
|
||||
Thread.sleep(2005);
|
||||
|
||||
expired = multipleConditionStorage.expired(tenant);
|
||||
expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeDeadline_Expired() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -200,17 +190,16 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults()).isEmpty();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dailyTimeWindow() {
|
||||
void dailyTimeWindow() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
LocalTime startTime = LocalTime.now().truncatedTo(ChronoUnit.MINUTES);
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -219,16 +208,15 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void slidingWindow() throws Exception {
|
||||
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
|
||||
|
||||
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
|
||||
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
|
||||
@@ -237,13 +225,13 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
|
||||
assertThat(window.getResults().get("a")).isTrue();
|
||||
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
|
||||
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
|
||||
assertThat(expired.size()).isZero();
|
||||
}
|
||||
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(String tenantId, TimeWindow sla) {
|
||||
private static Pair<Flow, MultipleCondition> mockFlow(TimeWindow sla) {
|
||||
var multipleCondition = MultipleCondition.builder()
|
||||
.id("condition-multiple-%s".formatted(tenantId))
|
||||
.id("condition-multiple")
|
||||
.conditions(ImmutableMap.of(
|
||||
"flow-a", ExecutionFlow.builder()
|
||||
.flowId(Property.ofValue("flow-a"))
|
||||
@@ -260,7 +248,6 @@ public abstract class AbstractMultipleConditionStorageTest {
|
||||
Flow flow = Flow.builder()
|
||||
.namespace(NAMESPACE)
|
||||
.id("multiple-flow")
|
||||
.tenantId(tenantId)
|
||||
.revision(1)
|
||||
.triggers(Collections.singletonList(io.kestra.plugin.core.trigger.Flow.builder()
|
||||
.id("trigger-flow")
|
||||
|
||||
@@ -13,20 +13,21 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
public abstract class AbstractFeatureUsageReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
FeatureUsageReport featureUsageReport;
|
||||
|
||||
|
||||
@Test
|
||||
public void shouldGetReport() {
|
||||
// When
|
||||
Instant now = Instant.now();
|
||||
FeatureUsageReport.UsageEvent event = featureUsageReport.report(
|
||||
now,
|
||||
now,
|
||||
Reportable.TimeInterval.of(now.minus(Duration.ofDays(1)).atZone(ZoneId.systemDefault()), now.atZone(ZoneId.systemDefault()))
|
||||
);
|
||||
|
||||
|
||||
// Then
|
||||
assertThat(event.getExecutions().getDailyExecutionsCount().size()).isGreaterThan(0);
|
||||
assertThat(event.getExecutions().getDailyTaskRunsCount()).isNull();
|
||||
}
|
||||
}
|
||||
@@ -18,10 +18,10 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
class SystemInformationReportTest {
|
||||
|
||||
|
||||
@Inject
|
||||
private SystemInformationReport systemInformationReport;
|
||||
|
||||
|
||||
@Test
|
||||
void shouldGetReport() {
|
||||
SystemInformationReport.SystemInformationEvent event = systemInformationReport.report(Instant.now());
|
||||
@@ -32,34 +32,34 @@ class SystemInformationReportTest {
|
||||
assertThat(event.host().getHardware().getLogicalProcessorCount()).isNotNull();
|
||||
assertThat(event.host().getJvm().getName()).isNotNull();
|
||||
assertThat(event.host().getOs().getFamily()).isNotNull();
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("h2");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("h2");
|
||||
assertThat(event.configurations().getRepositoryType()).isEqualTo("memory");
|
||||
assertThat(event.configurations().getQueueType()).isEqualTo("memory");
|
||||
}
|
||||
|
||||
|
||||
@MockBean(SettingRepositoryInterface.class)
|
||||
@Singleton
|
||||
static class TestSettingRepository implements SettingRepositoryInterface {
|
||||
public static Object UUID = null;
|
||||
|
||||
|
||||
@Override
|
||||
public Optional<Setting> findByKey(String key) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Setting> findAll() {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting save(Setting setting) throws ConstraintViolationException {
|
||||
if (setting.getKey().equals(Setting.INSTANCE_UUID)) {
|
||||
UUID = setting.getValue();
|
||||
}
|
||||
|
||||
|
||||
return setting;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Setting delete(Setting setting) {
|
||||
return setting;
|
||||
|
||||
@@ -25,7 +25,6 @@ import io.kestra.core.models.tasks.ResolvedTask;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.NamespaceUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
@@ -49,6 +48,7 @@ import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
@@ -62,17 +62,17 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId) {
|
||||
return builder(tenantId, state, flowId, NAMESPACE);
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId) {
|
||||
return builder(state, flowId, NAMESPACE);
|
||||
}
|
||||
|
||||
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId, String namespace) {
|
||||
public static Execution.ExecutionBuilder builder(State.Type state, String flowId, String namespace) {
|
||||
State finalState = randomDuration(state);
|
||||
|
||||
Execution.ExecutionBuilder execution = Execution.builder()
|
||||
.id(FriendlyId.createFriendlyId())
|
||||
.namespace(namespace)
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId(flowId == null ? FLOW : flowId)
|
||||
.flowRevision(1)
|
||||
.state(finalState);
|
||||
@@ -126,11 +126,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
return finalState;
|
||||
}
|
||||
|
||||
protected void inject(String tenantId) {
|
||||
inject(tenantId, null);
|
||||
protected void inject() {
|
||||
inject(null);
|
||||
}
|
||||
|
||||
protected void inject(String tenantId, String executionTriggerId) {
|
||||
protected void inject(String executionTriggerId) {
|
||||
ExecutionTrigger executionTrigger = null;
|
||||
|
||||
if (executionTriggerId != null) {
|
||||
@@ -139,7 +139,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
executionRepository.save(builder(tenantId, State.Type.RUNNING, null)
|
||||
executionRepository.save(builder(State.Type.RUNNING, null)
|
||||
.labels(List.of(
|
||||
new Label("key", "value"),
|
||||
new Label("key2", "value2")
|
||||
@@ -149,7 +149,6 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
);
|
||||
for (int i = 1; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).trigger(executionTrigger).build());
|
||||
@@ -157,7 +156,6 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
// add a test execution, this should be ignored in search & statistics
|
||||
executionRepository.save(builder(
|
||||
tenantId,
|
||||
State.Type.SUCCESS,
|
||||
null
|
||||
)
|
||||
@@ -169,10 +167,9 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter, int expectedSize){
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, "executionTriggerId");
|
||||
inject("executionTriggerId");
|
||||
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(expectedSize);
|
||||
}
|
||||
@@ -195,8 +192,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter)));
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -212,10 +208,9 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void find() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
inject();
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -224,7 +219,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value( List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -232,7 +227,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(1L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -240,7 +235,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value2"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -249,7 +244,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(Map.of("key", "value", "keyTest", "valueTest"))
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(0L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -257,7 +252,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value("second")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -271,7 +266,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(NAMESPACE)
|
||||
.build()
|
||||
);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(13L);
|
||||
|
||||
filters = List.of(QueryFilter.builder()
|
||||
@@ -279,7 +274,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.STARTS_WITH)
|
||||
.value("io.kestra")
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
}
|
||||
|
||||
@@ -287,16 +282,15 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
protected void findTriggerExecutionId() {
|
||||
String executionTriggerId = IdUtils.create();
|
||||
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant, executionTriggerId);
|
||||
inject(tenant);
|
||||
inject(executionTriggerId);
|
||||
inject();
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.TRIGGER_EXECUTION_ID)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(executionTriggerId)
|
||||
.build());
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -306,7 +300,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.CHILD)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
|
||||
@@ -317,21 +311,20 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.value(ExecutionRepositoryInterface.ChildFilter.MAIN)
|
||||
.build());
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters );
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters );
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
assertThat(executions.getFirst().getTrigger()).isNull();
|
||||
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(56L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findWithSort() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
inject();
|
||||
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), tenant, null);
|
||||
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), MAIN_TENANT, null);
|
||||
assertThat(executions.getTotal()).isEqualTo(28L);
|
||||
assertThat(executions.size()).isEqualTo(10);
|
||||
|
||||
@@ -340,107 +333,114 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(List.of(State.Type.RUNNING, State.Type.FAILED))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.getTotal()).isEqualTo(8L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findById() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
protected void findTaskRun() {
|
||||
inject();
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
ArrayListTotal<TaskRun> taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, null);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(74L);
|
||||
assertThat(taskRuns.size()).isEqualTo(10);
|
||||
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
|
||||
taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(taskRuns.getTotal()).isEqualTo(1L);
|
||||
assertThat(taskRuns.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
protected void findById() {
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(execution1.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_1.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByIdTestExecution() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var executionTest = ExecutionFixture.EXECUTION_TEST(tenant);
|
||||
executionRepository.save(executionTest);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_TEST);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(tenant, executionTest.getId());
|
||||
Optional<Execution> full = executionRepository.findById(null, ExecutionFixture.EXECUTION_TEST.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
assertThat(full.get().getId()).isEqualTo(executionTest.getId());
|
||||
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_TEST.getId());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void purge() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.purge(execution1);
|
||||
executionRepository.purge(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
full = executionRepository.findById(null, ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void purgeExecutions() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
var execution1 = ExecutionFixture.EXECUTION_1;
|
||||
executionRepository.save(execution1);
|
||||
var execution2 = ExecutionFixture.EXECUTION_2(tenant);
|
||||
var execution2 = ExecutionFixture.EXECUTION_2;
|
||||
executionRepository.save(execution2);
|
||||
|
||||
var results = executionRepository.purge(List.of(execution1, execution2));
|
||||
assertThat(results).isEqualTo(2);
|
||||
|
||||
assertThat(executionRepository.findById(tenant, execution1.getId())).isEmpty();
|
||||
assertThat(executionRepository.findById(tenant, execution2.getId())).isEmpty();
|
||||
assertThat(executionRepository.findById(null, execution1.getId())).isEmpty();
|
||||
assertThat(executionRepository.findById(null, execution2.getId())).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void delete() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution1);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
|
||||
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
executionRepository.delete(execution1);
|
||||
executionRepository.delete(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
full = executionRepository.findById(tenant, execution1.getId());
|
||||
full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
|
||||
assertThat(full.isPresent()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void mappingConflict() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2(tenant));
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1(tenant));
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_2);
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(tenant, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(MAIN_TENANT, NAMESPACE, FLOW, Pageable.from(1, 10));
|
||||
|
||||
assertThat(page1.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void dailyStatistics() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
@@ -450,14 +450,15 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
false);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
@@ -469,52 +470,131 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
false);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(21L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
false);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(20L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
false);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void taskRunsDailyStatistics() {
|
||||
for (int i = 0; i < 28; i++) {
|
||||
executionRepository.save(builder(
|
||||
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
|
||||
i < 15 ? null : "second"
|
||||
).build());
|
||||
}
|
||||
|
||||
executionRepository.save(builder(
|
||||
State.Type.SUCCESS,
|
||||
"second"
|
||||
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
|
||||
|
||||
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getDuration().getAvg().toMillis()).isGreaterThan(0L);
|
||||
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED)).isEqualTo(3L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.RUNNING)).isEqualTo(5L * 2);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER, FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.USER),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(55L);
|
||||
|
||||
result = executionRepository.dailyStatistics(
|
||||
null,
|
||||
MAIN_TENANT,
|
||||
List.of(FlowScope.SYSTEM),
|
||||
null,
|
||||
null,
|
||||
ZonedDateTime.now().minusDays(10),
|
||||
ZonedDateTime.now(),
|
||||
null,
|
||||
null,
|
||||
true);
|
||||
assertThat(result.size()).isEqualTo(11);
|
||||
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@SuppressWarnings("OptionalGetWithoutIsPresent")
|
||||
@Test
|
||||
protected void executionsCount() throws InterruptedException {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
for (int i = 0; i < 14; i++) {
|
||||
executionRepository.save(builder(
|
||||
tenant,
|
||||
State.Type.SUCCESS,
|
||||
i < 2 ? "first" : (i < 5 ? "second" : "third")
|
||||
).build());
|
||||
@@ -524,7 +604,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
Thread.sleep(500);
|
||||
|
||||
List<ExecutionCount> result = executionRepository.executionCounts(
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -543,7 +623,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("missing")).findFirst().get().getCount()).isEqualTo(0L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
new Flow(NAMESPACE, "first"),
|
||||
new Flow(NAMESPACE, "second"),
|
||||
@@ -560,7 +640,7 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("third")).findFirst().get().getCount()).isEqualTo(9L);
|
||||
|
||||
result = executionRepository.executionCounts(
|
||||
tenant,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
@@ -573,15 +653,14 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void update() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution execution = ExecutionFixture.EXECUTION_1(tenant);
|
||||
executionRepository.save(execution);
|
||||
Execution execution = ExecutionFixture.EXECUTION_1;
|
||||
executionRepository.save(ExecutionFixture.EXECUTION_1);
|
||||
|
||||
Label label = new Label("key", "value");
|
||||
Execution updated = execution.toBuilder().labels(List.of(label)).build();
|
||||
executionRepository.update(updated);
|
||||
|
||||
Optional<Execution> validation = executionRepository.findById(tenant, updated.getId());
|
||||
Optional<Execution> validation = executionRepository.findById(MAIN_TENANT, updated.getId());
|
||||
assertThat(validation.isPresent()).isTrue();
|
||||
assertThat(validation.get().getLabels().size()).isEqualTo(1);
|
||||
assertThat(validation.get().getLabels().getFirst()).isEqualTo(label);
|
||||
@@ -589,14 +668,13 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldFindLatestExecutionGivenState() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Execution earliest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(5)));
|
||||
Execution earliest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(10)));
|
||||
Execution latest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(5)));
|
||||
|
||||
executionRepository.save(earliest);
|
||||
executionRepository.save(latest);
|
||||
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(tenant, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
Optional<Execution> result = executionRepository.findLatestForStates(MAIN_TENANT, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
|
||||
assertThat(result.isPresent()).isTrue();
|
||||
assertThat(result.get().getId()).isEqualTo(latest.getId());
|
||||
}
|
||||
@@ -636,11 +714,11 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
assertThat(data.get(0).get("date")).isEqualTo(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").format(ZonedDateTime.ofInstant(startDate, ZoneId.systemDefault()).withSecond(0).withNano(0)));
|
||||
}
|
||||
|
||||
private static Execution buildWithCreatedDate(String tenant, Instant instant) {
|
||||
private static Execution buildWithCreatedDate(Instant instant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State(State.Type.CREATED, List.of(new State.History(State.Type.CREATED, instant))))
|
||||
@@ -651,24 +729,22 @@ public abstract class AbstractExecutionRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void findAllAsync() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
inject();
|
||||
|
||||
List<Execution> executions = executionRepository.findAllAsync(tenant).collectList().block();
|
||||
List<Execution> executions = executionRepository.findAllAsync(MAIN_TENANT).collectList().block();
|
||||
assertThat(executions).hasSize(29); // used by the backup so it contains TEST executions
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldFindByLabel() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
inject();
|
||||
|
||||
List<QueryFilter> filters = List.of(QueryFilter.builder()
|
||||
.field(QueryFilter.Field.LABELS)
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value"))
|
||||
.build());
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.size()).isEqualTo(1L);
|
||||
|
||||
// Filtering by two pairs of labels, since now its a and behavior, it should not return anything
|
||||
@@ -677,72 +753,19 @@ inject(tenant);
|
||||
.operation(QueryFilter.Op.EQUALS)
|
||||
.value(Map.of("key", "value", "keyother", "valueother"))
|
||||
.build());
|
||||
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
|
||||
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(executions.size()).isEqualTo(0L);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastExecutionsWhenInputsAreNull() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
inject(tenant);
|
||||
inject();
|
||||
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(tenant, null);
|
||||
List<Execution> lastExecutions = executionRepository.lastExecutions(MAIN_TENANT, null);
|
||||
|
||||
assertThat(lastExecutions).isNotEmpty();
|
||||
Set<String> flowIds = lastExecutions.stream().map(Execution::getFlowId).collect(Collectors.toSet());
|
||||
assertThat(flowIds.size()).isEqualTo(lastExecutions.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldIncludeRunningExecutionsInLastExecutions() {
|
||||
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
// Create an older finished execution for flow "full"
|
||||
Instant older = Instant.now().minus(Duration.ofMinutes(10));
|
||||
State finishedState = new State(
|
||||
State.Type.SUCCESS,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, older.minus(Duration.ofMinutes(1))),
|
||||
new State.History(State.Type.SUCCESS, older)
|
||||
)
|
||||
);
|
||||
Execution finished = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.tenantId(tenant)
|
||||
.namespace(NAMESPACE)
|
||||
.flowId(FLOW)
|
||||
.flowRevision(1)
|
||||
.state(finishedState)
|
||||
.taskRunList(List.of())
|
||||
.build();
|
||||
executionRepository.save(finished);
|
||||
|
||||
// Create a newer running execution for the same flow
|
||||
Instant newer = Instant.now().minus(Duration.ofMinutes(2));
|
||||
State runningState = new State(
|
||||
State.Type.RUNNING,
|
||||
List.of(
|
||||
new State.History(State.Type.CREATED, newer),
|
||||
new State.History(State.Type.RUNNING, newer)
|
||||
)
|
||||
);
|
||||
Execution running = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.tenantId(tenant)
|
||||
.namespace(NAMESPACE)
|
||||
.flowId(FLOW)
|
||||
.flowRevision(1)
|
||||
.state(runningState)
|
||||
.taskRunList(List.of())
|
||||
.build();
|
||||
executionRepository.save(running);
|
||||
|
||||
List<Execution> last = executionRepository.lastExecutions(tenant, null);
|
||||
|
||||
// Ensure we have one per flow and that for FLOW it is the running execution
|
||||
Map<String, Execution> byFlow = last.stream().collect(Collectors.toMap(Execution::getFlowId, e -> e));
|
||||
assertThat(byFlow.get(FLOW)).isNotNull();
|
||||
assertThat(byFlow.get(FLOW).getId()).isEqualTo(running.getId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.Helpers;
|
||||
import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.exceptions.InvalidQueryFiltersException;
|
||||
@@ -9,6 +10,7 @@ import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.QueryFilter.Field;
|
||||
import io.kestra.core.models.QueryFilter.Op;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.conditions.ConditionContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.ExecutionTrigger;
|
||||
@@ -18,6 +20,7 @@ import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
import io.kestra.core.models.triggers.PollingTriggerInterface;
|
||||
import io.kestra.core.models.triggers.TriggerContext;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.services.FlowService;
|
||||
import io.kestra.core.utils.Await;
|
||||
@@ -26,18 +29,22 @@ import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.slf4j.event.Level;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
@@ -45,14 +52,16 @@ import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
// If some counts are wrong in this test it means that one of the tests is not properly deleting what it created
|
||||
@KestraTest
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public abstract class AbstractFlowRepositoryTest {
|
||||
public static final String TEST_TENANT_ID = "tenant";
|
||||
public static final String TEST_NAMESPACE = "io.kestra.unittest";
|
||||
public static final String TEST_FLOW_ID = "test";
|
||||
@Inject
|
||||
@@ -61,18 +70,21 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@BeforeAll
|
||||
protected static void init() {
|
||||
@Inject
|
||||
private LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
TestsUtils.loads(MAIN_TENANT, repositoryLoader);
|
||||
FlowListener.reset();
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, IdUtils.create(), TEST_FLOW_ID);
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder() {
|
||||
return builder(IdUtils.create(), TEST_FLOW_ID);
|
||||
}
|
||||
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId, String flowId, String taskId) {
|
||||
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String flowId, String taskId) {
|
||||
return FlowWithSource.builder()
|
||||
.tenantId(tenantId)
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tasks(Collections.singletonList(Return.builder().id(taskId).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()));
|
||||
@@ -81,16 +93,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -101,16 +113,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_with_source(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id("filterFlowId")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.labels(Label.from(Map.of("key", "value")))
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
} finally {
|
||||
@@ -132,7 +144,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
() -> flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -141,7 +153,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
void should_fail_to_find_all_with_source(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
() -> flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -164,17 +176,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId());
|
||||
Optional<Flow> full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -183,18 +195,17 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithoutAcl() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.revision(3)
|
||||
.build();
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
Optional<Flow> full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getRevision()).isEqualTo(1);
|
||||
|
||||
full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(flow);
|
||||
@@ -203,16 +214,15 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByIdWithSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant)
|
||||
.tenantId(tenant)
|
||||
FlowWithSource flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.revision(3)
|
||||
.build();
|
||||
String source = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, source));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, source));
|
||||
|
||||
try {
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(tenant, flow.getNamespace(), flow.getId());
|
||||
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
|
||||
full.ifPresent(current -> {
|
||||
@@ -227,8 +237,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).revision(12).build();
|
||||
FlowWithSource flow = builder().revision(12).build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -240,8 +249,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void saveNoRevision() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flow = builder(tenant).build();
|
||||
FlowWithSource flow = builder().build();
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
@@ -252,17 +260,56 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
List<Flow> save = flowRepository.findAll(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSource(MAIN_TENANT);
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
List<Flow> save = flowRepository.findAllForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllWithSourceForAllTenants() {
|
||||
List<FlowWithSource> save = flowRepository.findAllWithSourceForAllTenants();
|
||||
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefix() {
|
||||
List<Flow> save = flowRepository.findByNamespacePrefix(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespaceWithSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
Flow flow = builder()
|
||||
.revision(3)
|
||||
.build();
|
||||
String flowSource = "# comment\n" + flow.sourceOrGenerateIfNull();
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(tenant, flowSource));
|
||||
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, flowSource));
|
||||
|
||||
try {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(tenant, flow.getNamespace());
|
||||
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(MAIN_TENANT, flow.getNamespace());
|
||||
assertThat((long) save.size()).isEqualTo(1L);
|
||||
|
||||
assertThat(save.getFirst().getSource()).isEqualTo(FlowService.cleanupSource(flowSource));
|
||||
@@ -271,15 +318,175 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespacePrefixWithSource() {
|
||||
List<FlowWithSource> save = flowRepository.findByNamespacePrefixWithSource(MAIN_TENANT, "io.kestra.tests");
|
||||
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationPartial() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating at MAX-1, it should return MAX-1")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT - 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_paginationGreaterThanExisting() {
|
||||
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
|
||||
.size())
|
||||
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_prefixMatchingAllNamespaces() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
|
||||
)
|
||||
).size())
|
||||
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
|
||||
.isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecifiedNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
|
||||
)
|
||||
).size()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificSubNamespace() {
|
||||
assertThat(flowRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
|
||||
assertThat(flowRepository.findWithSource(
|
||||
Pageable.UNPAGED,
|
||||
MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_aSpecificFlowByNamespaceAndLabel() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find_noResult_forAnUnknownNamespace() {
|
||||
assertThat(
|
||||
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
|
||||
assertThat(
|
||||
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
|
||||
List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
|
||||
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
|
||||
)
|
||||
).size())
|
||||
.isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void findSpecialChars() {
|
||||
ArrayListTotal<SearchResult<Flow>> save = flowRepository.findSourceCode(Pageable.unpaged(), "https://api.chucknorris.io", MAIN_TENANT, null);
|
||||
assertThat((long) save.size()).isEqualTo(2L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void delete() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant).tenantId(tenant).build();
|
||||
Flow flow = builder().tenantId(MAIN_TENANT).build();
|
||||
|
||||
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(tenant, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, save.getNamespace(), save.getId()).isPresent()).isTrue();
|
||||
} catch (Throwable e) {
|
||||
deleteFlow(save);
|
||||
throw e;
|
||||
@@ -287,22 +494,21 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow delete = flowRepository.delete(save);
|
||||
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
|
||||
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(tenant, flow.getNamespace(), flow.getId());
|
||||
List<FlowWithSource> revisions = flowRepository.findRevisions(MAIN_TENANT, flow.getNamespace(), flow.getId());
|
||||
assertThat(revisions.getLast().getRevision()).isEqualTo(delete.getRevision());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateConflict() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("a").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -310,12 +516,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
|
||||
try {
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest2")
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.inputs(List.of(StringInput.builder().type(Type.STRING).id("b").build()))
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
@@ -333,14 +539,13 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void removeTrigger() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
void removeTrigger() throws TimeoutException, QueueException {
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -350,12 +555,12 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
flow = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
|
||||
Flow update = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
|
||||
.build();
|
||||
;
|
||||
@@ -366,25 +571,21 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
deleteFlow(flow);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.getEmits().size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void removeTriggerDelete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String flowId = IdUtils.create();
|
||||
|
||||
Flow flow = Flow.builder()
|
||||
.id(flowId)
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.triggers(Collections.singletonList(UnitTest.builder()
|
||||
.id("sleep")
|
||||
.type(UnitTest.class.getName())
|
||||
@@ -394,39 +595,34 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
Flow save = flowRepository.create(GenericFlow.of(flow));
|
||||
try {
|
||||
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
|
||||
} finally {
|
||||
deleteFlow(save);
|
||||
}
|
||||
|
||||
Await.until(() -> FlowListener.filterByTenant(tenant)
|
||||
.size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.filterByTenant(tenant).stream()
|
||||
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
Await.until(() -> FlowListener.getEmits().size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
protected void shouldReturnNullRevisionForNonExistingFlow() {
|
||||
assertThat(flowRepository.lastRevision(TestsUtils.randomTenant(this.getClass().getSimpleName()), TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, IdUtils.create())).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldReturnLastRevisionOnCreate() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId);
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "???")));
|
||||
Integer result = flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId);
|
||||
|
||||
// Then
|
||||
assertThat(result).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -435,36 +631,34 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final String flowId = IdUtils.create();
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
|
||||
|
||||
// When
|
||||
flowRepository.delete(created);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnCreateAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
flowRepository.delete(
|
||||
flowRepository.create(createTestingLogFlow(tenant, flowId, "first"))
|
||||
flowRepository.create(createTestingLogFlow(flowId, "first"))
|
||||
);
|
||||
|
||||
// When
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "second")));
|
||||
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "second")));
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -473,23 +667,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldReturnNullForLastRevisionAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isNull();
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isNull();
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -498,23 +691,22 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
@Test
|
||||
protected void shouldFindAllRevisionsAfterDelete() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// When
|
||||
flowRepository.delete(updated);
|
||||
|
||||
// Then
|
||||
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
|
||||
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
@@ -522,22 +714,21 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldIncrementRevisionOnUpdateGivenNotEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(2);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
@@ -546,39 +737,48 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
protected void shouldNotIncrementRevisionOnUpdateGivenEqualSource() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
final List<Flow> toDelete = new ArrayList<>();
|
||||
final String flowId = IdUtils.create();
|
||||
try {
|
||||
|
||||
// Given
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
|
||||
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
|
||||
toDelete.add(created);
|
||||
|
||||
// When
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "first"), created);
|
||||
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "first"), created);
|
||||
toDelete.add(updated);
|
||||
|
||||
// Then
|
||||
assertThat(updated.getRevision()).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
|
||||
|
||||
} finally {
|
||||
toDelete.forEach(this::deleteFlow);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnForGivenQueryWildCardFilters() {
|
||||
List<QueryFilter> filters = List.of(
|
||||
QueryFilter.builder().field(QueryFilter.Field.QUERY).operation(QueryFilter.Op.EQUALS).value("*").build()
|
||||
);
|
||||
ArrayListTotal<Flow> flows = flowRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
|
||||
assertThat(flows.size()).isEqualTo(10);
|
||||
assertThat(flows.getTotal()).isEqualTo(Helpers.FLOWS_COUNT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByExecution() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
Flow flow = builder()
|
||||
.tenantId(MAIN_TENANT)
|
||||
.revision(1)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.tenantId(tenant)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId(flow.getId())
|
||||
.flowRevision(flow.getRevision())
|
||||
.state(new State())
|
||||
@@ -603,13 +803,11 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByExecutionNoRevision() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Flow flow = builder(tenant)
|
||||
Flow flow = builder()
|
||||
.revision(3)
|
||||
.build();
|
||||
flowRepository.create(GenericFlow.of(flow));
|
||||
Execution execution = Execution.builder()
|
||||
.tenantId(tenant)
|
||||
.id(IdUtils.create())
|
||||
.namespace(flow.getNamespace())
|
||||
.flowId(flow.getId())
|
||||
@@ -635,17 +833,16 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource toDelete = null;
|
||||
try {
|
||||
// Given
|
||||
Flow flow = createTestFlowForNamespace(tenant, TEST_NAMESPACE);
|
||||
Flow flow = createTestFlowForNamespace(TEST_NAMESPACE);
|
||||
toDelete = flowRepository.create(GenericFlow.of(flow));
|
||||
// When
|
||||
int count = flowRepository.count(tenant);
|
||||
int count = flowRepository.count(MAIN_TENANT);
|
||||
|
||||
// Then
|
||||
assertTrue(count > 0);
|
||||
Assertions.assertTrue(count > 0);
|
||||
} finally {
|
||||
Optional.ofNullable(toDelete).ifPresent(flow -> {
|
||||
flowRepository.delete(flow);
|
||||
@@ -653,41 +850,11 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void should_exist_for_tenant(){
|
||||
String tenantFlowExist = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flowExist = FlowWithSource.builder()
|
||||
.id("flowExist")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenantFlowExist)
|
||||
.deleted(false)
|
||||
.build();
|
||||
flowExist = flowRepository.create(GenericFlow.of(flowExist));
|
||||
|
||||
String tenantFlowDeleted = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
FlowWithSource flowDeleted = FlowWithSource.builder()
|
||||
.id("flowDeleted")
|
||||
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
|
||||
.tenantId(tenantFlowDeleted)
|
||||
.deleted(true)
|
||||
.build();
|
||||
flowDeleted = flowRepository.create(GenericFlow.of(flowDeleted));
|
||||
|
||||
try {
|
||||
assertTrue(flowRepository.existAnyNoAcl(tenantFlowExist));
|
||||
assertFalse(flowRepository.existAnyNoAcl("not_found"));
|
||||
assertFalse(flowRepository.existAnyNoAcl(tenantFlowDeleted));
|
||||
} finally {
|
||||
deleteFlow(flowExist);
|
||||
deleteFlow(flowDeleted);
|
||||
}
|
||||
}
|
||||
|
||||
private static Flow createTestFlowForNamespace(String tenantId, String namespace) {
|
||||
private static Flow createTestFlowForNamespace(String namespace) {
|
||||
return Flow.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace)
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.tasks(List.of(Return.builder()
|
||||
.id(IdUtils.create())
|
||||
.type(Return.class.getName())
|
||||
@@ -706,31 +873,21 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<AbstractFlow>> {
|
||||
private static List<CrudEvent<AbstractFlow>> emits = new CopyOnWriteArrayList<>();
|
||||
public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
|
||||
@Getter
|
||||
private static List<CrudEvent<Flow>> emits = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<AbstractFlow> event) {
|
||||
//This has to be done because Micronaut may send CrudEvent<Setting> for example, and we don't want them.
|
||||
if ((event.getModel() != null && event.getModel() instanceof AbstractFlow)||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof AbstractFlow)) {
|
||||
emits.add(event);
|
||||
}
|
||||
public void onApplicationEvent(CrudEvent<Flow> event) {
|
||||
emits.add(event);
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
emits = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
public static List<CrudEvent<AbstractFlow>> filterByTenant(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getModel() != null && e.getModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
emits = new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
private static GenericFlow createTestingLogFlow(String tenantId, String id, String logMessage) {
|
||||
private static GenericFlow createTestingLogFlow(String id, String logMessage) {
|
||||
String source = """
|
||||
id: %s
|
||||
namespace: %s
|
||||
@@ -739,7 +896,7 @@ public abstract class AbstractFlowRepositoryTest {
|
||||
type: io.kestra.plugin.core.log.Log
|
||||
message: %s
|
||||
""".formatted(id, TEST_NAMESPACE, logMessage);
|
||||
return GenericFlow.fromYaml(tenantId, source);
|
||||
return GenericFlow.fromYaml(TEST_TENANT_ID, source);
|
||||
}
|
||||
|
||||
protected static int COUNTER = 0;
|
||||
|
||||
@@ -4,7 +4,7 @@ import io.kestra.core.models.topologies.FlowNode;
|
||||
import io.kestra.core.models.topologies.FlowRelation;
|
||||
import io.kestra.core.models.topologies.FlowTopology;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.core.tenant.TenantService;
|
||||
import jakarta.inject.Inject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@@ -17,21 +17,21 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
@Inject
|
||||
private FlowTopologyRepositoryInterface flowTopologyRepository;
|
||||
|
||||
protected FlowTopology createSimpleFlowTopology(String tenantId, String flowA, String flowB, String namespace) {
|
||||
protected FlowTopology createSimpleFlowTopology(String flowA, String flowB, String namespace) {
|
||||
return FlowTopology.builder()
|
||||
.relation(FlowRelation.FLOW_TASK)
|
||||
.source(FlowNode.builder()
|
||||
.id(flowA)
|
||||
.namespace(namespace)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowA)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowA)
|
||||
.build()
|
||||
)
|
||||
.destination(FlowNode.builder()
|
||||
.id(flowB)
|
||||
.namespace(namespace)
|
||||
.tenantId(tenantId)
|
||||
.uid(tenantId + flowB)
|
||||
.tenantId(TenantService.MAIN_TENANT)
|
||||
.uid(flowB)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
@@ -39,45 +39,42 @@ public abstract class AbstractFlowTopologyRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findByFlow() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(tenant, "io.kestra.tests", "flow-a", false);
|
||||
List<FlowTopology> list = flowTopologyRepository.findByFlow(TenantService.MAIN_TENANT, "io.kestra.tests", "flow-a", false);
|
||||
|
||||
assertThat(list.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(tenant, "io.kestra.tests");
|
||||
List<FlowTopology> list = flowTopologyRepository.findByNamespace(TenantService.MAIN_TENANT, "io.kestra.tests");
|
||||
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
|
||||
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
|
||||
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
|
||||
);
|
||||
flowTopologyRepository.save(
|
||||
createSimpleFlowTopology(tenant, "flow-e", "flow-f", "io.kestra.tests.2")
|
||||
createSimpleFlowTopology("flow-e", "flow-f", "io.kestra.tests.2")
|
||||
);
|
||||
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(tenant);
|
||||
List<FlowTopology> list = flowTopologyRepository.findAll(TenantService.MAIN_TENANT);
|
||||
|
||||
assertThat(list.size()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@@ -34,7 +34,9 @@ import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatReflectiveOperationException;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@@ -42,11 +44,11 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@Inject
|
||||
protected LogRepositoryInterface logRepository;
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level) {
|
||||
return logEntry(tenantId, level, IdUtils.create());
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level) {
|
||||
return logEntry(level, IdUtils.create());
|
||||
}
|
||||
|
||||
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level, String executionId) {
|
||||
protected static LogEntry.LogEntryBuilder logEntry(Level level, String executionId) {
|
||||
return LogEntry.builder()
|
||||
.flowId("flowId")
|
||||
.namespace("io.kestra.unittest")
|
||||
@@ -57,7 +59,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
.timestamp(Instant.now())
|
||||
.level(level)
|
||||
.thread("")
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.triggerId("triggerId")
|
||||
.message("john doe");
|
||||
}
|
||||
@@ -65,10 +67,9 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -76,10 +77,9 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_async(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAsync(tenant, List.of(filter));
|
||||
Flux<LogEntry> find = logRepository.findAsync(MAIN_TENANT, List.of(filter));
|
||||
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(1);
|
||||
@@ -88,12 +88,11 @@ public abstract class AbstractLogRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_delete_with_filter(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
|
||||
logRepository.save(logEntry(Level.INFO, "executionId").build());
|
||||
|
||||
logRepository.deleteByFilters(tenant, List.of(filter));
|
||||
logRepository.deleteByFilters(MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(logRepository.findAllAsync(tenant).collectList().block()).isEmpty();
|
||||
assertThat(logRepository.findAllAsync(MAIN_TENANT).collectList().block()).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
@@ -153,10 +152,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(
|
||||
InvalidQueryFiltersException.class,
|
||||
() -> logRepository.find(
|
||||
Pageable.UNPAGED,
|
||||
TestsUtils.randomTenant(this.getClass().getSimpleName()),
|
||||
List.of(filter)));
|
||||
() -> logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
|
||||
}
|
||||
|
||||
@@ -174,17 +170,16 @@ public abstract class AbstractLogRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
|
||||
LogEntry save = logRepository.save(builder.build());
|
||||
logRepository.save(builder.executionKind(ExecutionKind.TEST).build()); // should only be loaded by execution id
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
var filters = List.of(QueryFilter.builder()
|
||||
@@ -200,7 +195,7 @@ public abstract class AbstractLogRepositoryTest {
|
||||
find = logRepository.find(Pageable.UNPAGED, "doe", filters);
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
find = logRepository.find(Pageable.UNPAGED, tenant, null);
|
||||
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
@@ -208,146 +203,141 @@ public abstract class AbstractLogRepositoryTest {
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
List<LogEntry> list = logRepository.findByExecutionId(tenant, save.getExecutionId(), null);
|
||||
List<LogEntry> list = logRepository.findByExecutionId(MAIN_TENANT, save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
list = logRepository.findByExecutionId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(tenant, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null, 0);
|
||||
assertThat(list.size()).isEqualTo(2);
|
||||
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
Integer countDeleted = logRepository.purge(Execution.builder().id(save.getExecutionId()).build());
|
||||
assertThat(countDeleted).isEqualTo(2);
|
||||
|
||||
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
|
||||
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
|
||||
assertThat(list.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void pageable() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = "123";
|
||||
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
|
||||
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
|
||||
builder.executionId(executionId);
|
||||
|
||||
for (int i = 0; i < 80; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
builder = logEntry(tenant, Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
builder = logEntry(Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
|
||||
LogEntry logEntry2 = logRepository.save(builder.build());
|
||||
for (int i = 0; i < 20; i++) {
|
||||
logRepository.save(builder.build());
|
||||
}
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(50);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(3, 50));
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(3, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getTotal()).isEqualTo(101L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskId(tenant, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
|
||||
|
||||
assertThat(find.size()).isEqualTo(21);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
|
||||
|
||||
assertThat(find.size()).isEqualTo(10);
|
||||
assertThat(find.getTotal()).isEqualTo(21L);
|
||||
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
|
||||
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFindByExecutionIdTestLogs() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
var builder = logEntry(tenant, Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
var builder = logEntry(Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
|
||||
logRepository.save(builder);
|
||||
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(tenant, builder.getExecutionId(), null);
|
||||
List<LogEntry> logs = logRepository.findByExecutionId(MAIN_TENANT, builder.getExecutionId(), null);
|
||||
assertThat(logs).hasSize(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deleteByQuery() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
LogEntry log1 = logEntry(tenant, Level.INFO).build();
|
||||
LogEntry log1 = logEntry(Level.INFO).build();
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(tenant, log1.getExecutionId(), null, null, null, null);
|
||||
logRepository.deleteByQuery(MAIN_TENANT, log1.getExecutionId(), null, null, null, null);
|
||||
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null);
|
||||
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null);
|
||||
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
|
||||
logRepository.save(log1);
|
||||
|
||||
logRepository.deleteByQuery(tenant, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
logRepository.deleteByQuery(MAIN_TENANT, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
|
||||
|
||||
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
|
||||
assertThat(find.size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(tenant, Level.ERROR).build());
|
||||
logRepository.save(logEntry(tenant, Level.WARN).build());
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
logRepository.save(logEntry(Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
|
||||
logRepository.save(logEntry(Level.ERROR).build());
|
||||
logRepository.save(logEntry(Level.WARN).build());
|
||||
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(tenant);
|
||||
Flux<LogEntry> find = logRepository.findAllAsync(MAIN_TENANT);
|
||||
List<LogEntry> logEntries = find.collectList().block();
|
||||
assertThat(logEntries).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fetchData() throws IOException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO).build());
|
||||
logRepository.save(logEntry(Level.INFO).build());
|
||||
|
||||
var results = logRepository.fetchData(tenant,
|
||||
var results = logRepository.fetchData(MAIN_TENANT,
|
||||
Logs.builder()
|
||||
.type(Logs.class.getName())
|
||||
.columns(Map.of(
|
||||
@@ -363,11 +353,10 @@ public abstract class AbstractLogRepositoryTest {
|
||||
|
||||
@Test
|
||||
void purge() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution2").build());
|
||||
logRepository.save(logEntry(tenant, Level.INFO, "execution2").build());
|
||||
logRepository.save(logEntry(Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(Level.INFO, "execution1").build());
|
||||
logRepository.save(logEntry(Level.INFO, "execution2").build());
|
||||
logRepository.save(logEntry(Level.INFO, "execution2").build());
|
||||
|
||||
var result = logRepository.purge(List.of(Execution.builder().id("execution1").build(), Execution.builder().id("execution2").build()));
|
||||
assertThat(result).isEqualTo(4);
|
||||
|
||||
@@ -8,7 +8,6 @@ import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.executions.metrics.Counter;
|
||||
import io.kestra.core.models.executions.metrics.MetricAggregations;
|
||||
import io.kestra.core.models.executions.metrics.Timer;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -28,28 +27,27 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
MetricEntry testCounter = MetricEntry.of(taskRun1, counter("test"), ExecutionKind.TEST);
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(testCounter); // should only be retrieved by execution id
|
||||
metricRepository.save(timer);
|
||||
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(tenant, executionId, Pageable.from(1, 10));
|
||||
List<MetricEntry> results = metricRepository.findByExecutionId(null, executionId, Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskId(tenant, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskId(null, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(3);
|
||||
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(tenant, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
results = metricRepository.findByExecutionIdAndTaskRunId(null, executionId, taskRun1.getId(), Pageable.from(1, 10));
|
||||
assertThat(results.size()).isEqualTo(2);
|
||||
|
||||
MetricAggregations aggregationResults = metricRepository.aggregateByFlowId(
|
||||
tenant,
|
||||
null,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -63,7 +61,7 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
assertThat(aggregationResults.getGroupBy()).isEqualTo("day");
|
||||
|
||||
aggregationResults = metricRepository.aggregateByFlowId(
|
||||
tenant,
|
||||
null,
|
||||
"namespace",
|
||||
"flow",
|
||||
null,
|
||||
@@ -80,12 +78,11 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void names() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task2");
|
||||
TaskRun taskRun2 = taskRun(executionId, "task2");
|
||||
MetricEntry counter2 = MetricEntry.of(taskRun2, counter("counter2"), null);
|
||||
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
@@ -95,9 +92,9 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
metricRepository.save(test); // should only be retrieved by execution id
|
||||
|
||||
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(tenant, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(tenant, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(tenant, "namespace", "flow");
|
||||
List<String> flowMetricsNames = metricRepository.flowMetrics(null, "namespace", "flow");
|
||||
List<String> taskMetricsNames = metricRepository.taskMetrics(null, "namespace", "flow", "task");
|
||||
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(null, "namespace", "flow");
|
||||
|
||||
assertThat(flowMetricsNames.size()).isEqualTo(2);
|
||||
assertThat(taskMetricsNames.size()).isEqualTo(1);
|
||||
@@ -106,28 +103,26 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAllAsync() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
String executionId = FriendlyId.createFriendlyId();
|
||||
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
|
||||
TaskRun taskRun1 = taskRun(executionId, "task");
|
||||
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
|
||||
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
|
||||
TaskRun taskRun2 = taskRun(executionId, "task");
|
||||
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
|
||||
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
|
||||
metricRepository.save(counter);
|
||||
metricRepository.save(timer);
|
||||
metricRepository.save(test); // should be retrieved as findAllAsync is used for backup
|
||||
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(tenant).collectList().block();
|
||||
List<MetricEntry> results = metricRepository.findAllAsync(null).collectList().block();
|
||||
assertThat(results).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void purge() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution1", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution1", "task"), counter("counter2"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution2", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun(tenant, "execution2", "task"), counter("counter2"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun("execution1", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun("execution1", "task"), counter("counter2"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun("execution2", "task"), counter("counter1"), null));
|
||||
metricRepository.save(MetricEntry.of(taskRun("execution2", "task"), counter("counter2"), null));
|
||||
|
||||
var result = metricRepository.purge(List.of(Execution.builder().id("execution1").build(), Execution.builder().id("execution2").build()));
|
||||
assertThat(result).isEqualTo(4);
|
||||
@@ -141,9 +136,8 @@ public abstract class AbstractMetricRepositoryTest {
|
||||
return Timer.of("counter", Duration.ofSeconds(5));
|
||||
}
|
||||
|
||||
private TaskRun taskRun(String tenantId, String executionId, String taskId) {
|
||||
private TaskRun taskRun(String executionId, String taskId) {
|
||||
return TaskRun.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId("flow")
|
||||
.namespace("namespace")
|
||||
.executionId(executionId)
|
||||
|
||||
@@ -4,8 +4,6 @@ import io.kestra.core.events.CrudEvent;
|
||||
import io.kestra.core.events.CrudEventType;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.templates.Template;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
@@ -13,10 +11,7 @@ import io.micronaut.data.model.Pageable;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
@@ -25,8 +20,6 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -35,60 +28,55 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
@Inject
|
||||
protected TemplateRepositoryInterface templateRepository;
|
||||
|
||||
@BeforeAll
|
||||
protected static void init() throws IOException, URISyntaxException {
|
||||
@BeforeEach
|
||||
protected void init() throws IOException, URISyntaxException {
|
||||
TemplateListener.reset();
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId) {
|
||||
return builder(tenantId, null);
|
||||
protected static Template.TemplateBuilder<?, ?> builder() {
|
||||
return builder(null);
|
||||
}
|
||||
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String tenantId, String namespace) {
|
||||
protected static Template.TemplateBuilder<?, ?> builder(String namespace) {
|
||||
return Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace(namespace == null ? "kestra.test" : namespace)
|
||||
.tenantId(tenantId)
|
||||
.tasks(Collections.singletonList(Return.builder().id("test").type(Return.class.getName()).format(Property.ofValue("test")).build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void findById() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
Template template = builder().build();
|
||||
templateRepository.create(template);
|
||||
|
||||
Optional<Template> full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
Optional<Template> full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
|
||||
full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
|
||||
full = templateRepository.findById(null, template.getNamespace(), template.getId());
|
||||
assertThat(full.isPresent()).isTrue();
|
||||
assertThat(full.get().getId()).isEqualTo(template.getId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByNamespace() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
Template template1 = builder().build();
|
||||
Template template2 = Template.builder()
|
||||
.id(IdUtils.create())
|
||||
.tenantId(tenant)
|
||||
.namespace("kestra.test.template").build();
|
||||
|
||||
templateRepository.create(template1);
|
||||
templateRepository.create(template2);
|
||||
|
||||
List<Template> templates = templateRepository.findByNamespace(tenant, template1.getNamespace());
|
||||
List<Template> templates = templateRepository.findByNamespace(null, template1.getNamespace());
|
||||
assertThat(templates.size()).isGreaterThanOrEqualTo(1);
|
||||
templates = templateRepository.findByNamespace(tenant, template2.getNamespace());
|
||||
templates = templateRepository.findByNamespace(null, template2.getNamespace());
|
||||
assertThat(templates.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void save() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
Template template = builder().build();
|
||||
Template save = templateRepository.create(template);
|
||||
|
||||
assertThat(save.getId()).isEqualTo(template.getId());
|
||||
@@ -96,42 +84,41 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
|
||||
@Test
|
||||
void findAll() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAll(tenant).size();
|
||||
Template template = builder(tenant).build();
|
||||
long saveCount = templateRepository.findAll(null).size();
|
||||
Template template = builder().build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAll(tenant).size();
|
||||
long size = templateRepository.findAll(null).size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAll(tenant).size()).isEqualTo(saveCount);
|
||||
assertThat((long) templateRepository.findAll(null).size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAllForAllTenants() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
long saveCount = templateRepository.findAllForAllTenants().size();
|
||||
Template template = builder(tenant).build();
|
||||
Template template = builder().build();
|
||||
templateRepository.create(template);
|
||||
long size = templateRepository.findAllForAllTenants().size();
|
||||
assertThat(size).isGreaterThan(saveCount);
|
||||
templateRepository.delete(template);
|
||||
assertThat((long) templateRepository.findAllForAllTenants().size()).isEqualTo(saveCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
void find() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template1 = builder(tenant).build();
|
||||
Template template1 = builder().build();
|
||||
templateRepository.create(template1);
|
||||
Template template2 = builder(tenant).build();
|
||||
Template template2 = builder().build();
|
||||
templateRepository.create(template2);
|
||||
Template template3 = builder(tenant).build();
|
||||
Template template3 = builder().build();
|
||||
templateRepository.create(template3);
|
||||
|
||||
// with pageable
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, tenant, "kestra.test");
|
||||
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, null, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
// without pageable
|
||||
save = templateRepository.find(null, tenant, "kestra.test");
|
||||
save = templateRepository.find(null, null, "kestra.test");
|
||||
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
|
||||
|
||||
templateRepository.delete(template1);
|
||||
@@ -139,45 +126,31 @@ public abstract class AbstractTemplateRepositoryTest {
|
||||
templateRepository.delete(template3);
|
||||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractTemplateRepositoryTest.class);
|
||||
|
||||
@Test
|
||||
protected void delete() throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Template template = builder(tenant).build();
|
||||
void delete() {
|
||||
Template template = builder().build();
|
||||
|
||||
Template save = templateRepository.create(template);
|
||||
templateRepository.delete(save);
|
||||
|
||||
assertThat(templateRepository.findById(tenant, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
assertThat(templateRepository.findById(null, template.getNamespace(), template.getId()).isPresent()).isFalse();
|
||||
|
||||
Await.until(() -> {
|
||||
LOG.info("-------------> number of event: {}", TemplateListener.getEmits(tenant).size());
|
||||
return TemplateListener.getEmits(tenant).size() == 2;
|
||||
|
||||
}, Duration.ofMillis(100), Duration.ofSeconds(5));
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits().size()).isEqualTo(2);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
|
||||
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
|
||||
}
|
||||
|
||||
@Singleton
|
||||
public static class TemplateListener implements ApplicationEventListener<CrudEvent<Template>> {
|
||||
private static List<CrudEvent<Template>> emits = new CopyOnWriteArrayList<>();
|
||||
private static List<CrudEvent<Template>> emits = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(CrudEvent<Template> event) {
|
||||
//The instanceOf is required because Micronaut may send non Template event via this method
|
||||
if ((event.getModel() != null && event.getModel() instanceof Template) ||
|
||||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof Template)) {
|
||||
emits.add(event);
|
||||
}
|
||||
emits.add(event);
|
||||
}
|
||||
|
||||
public static List<CrudEvent<Template>> getEmits(String tenantId){
|
||||
return emits.stream()
|
||||
.filter(e -> (e.getModel() != null && e.getModel().getTenantId().equals(tenantId)) ||
|
||||
(e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)))
|
||||
.toList();
|
||||
public static List<CrudEvent<Template>> getEmits() {
|
||||
return emits;
|
||||
}
|
||||
|
||||
public static void reset() {
|
||||
|
||||
@@ -9,7 +9,6 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.triggers.Trigger;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -25,6 +24,7 @@ import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.models.flows.FlowScope.USER;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@@ -35,9 +35,8 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Inject
|
||||
protected TriggerRepositoryInterface triggerRepository;
|
||||
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger(String tenantId) {
|
||||
private static Trigger.TriggerBuilder<?, ?> trigger() {
|
||||
return Trigger.builder()
|
||||
.tenantId(tenantId)
|
||||
.flowId(IdUtils.create())
|
||||
.namespace(TEST_NAMESPACE)
|
||||
.triggerId(IdUtils.create())
|
||||
@@ -45,9 +44,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
.date(ZonedDateTime.now());
|
||||
}
|
||||
|
||||
protected static Trigger generateDefaultTrigger(String tenantId){
|
||||
protected static Trigger generateDefaultTrigger(){
|
||||
Trigger trigger = Trigger.builder()
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.triggerId("triggerId")
|
||||
.namespace("trigger.namespace")
|
||||
.flowId("flowId")
|
||||
@@ -60,10 +59,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
|
||||
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -71,10 +69,9 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("filterCombinations")
|
||||
void should_find_all_async(QueryFilter filter){
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(generateDefaultTrigger(tenant));
|
||||
triggerRepository.save(generateDefaultTrigger());
|
||||
|
||||
List<Trigger> entries = triggerRepository.find(tenant, List.of(filter)).collectList().block();
|
||||
List<Trigger> entries = triggerRepository.find(MAIN_TENANT, List.of(filter)).collectList().block();
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
}
|
||||
@@ -95,7 +92,7 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@ParameterizedTest
|
||||
@MethodSource("errorFilterCombinations")
|
||||
void should_fail_to_find_all(QueryFilter filter){
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
|
||||
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
|
||||
}
|
||||
|
||||
static Stream<QueryFilter> errorFilterCombinations() {
|
||||
@@ -113,8 +110,7 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
|
||||
@Test
|
||||
void all() {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger(tenant);
|
||||
Trigger.TriggerBuilder<?, ?> builder = trigger();
|
||||
|
||||
Optional<Trigger> findLast = triggerRepository.findLast(builder.build());
|
||||
assertThat(findLast.isPresent()).isFalse();
|
||||
@@ -134,47 +130,47 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
assertThat(findLast.get().getExecutionId()).isEqualTo(save.getExecutionId());
|
||||
|
||||
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
triggerRepository.save(trigger(tenant).build());
|
||||
Trigger searchedTrigger = trigger(tenant).build();
|
||||
triggerRepository.save(trigger().build());
|
||||
triggerRepository.save(trigger().build());
|
||||
Trigger searchedTrigger = trigger().build();
|
||||
triggerRepository.save(searchedTrigger);
|
||||
|
||||
List<Trigger> all = triggerRepository.findAllForAllTenants();
|
||||
|
||||
assertThat(all.size()).isGreaterThanOrEqualTo(4);
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
|
||||
all = triggerRepository.findAll(tenant);
|
||||
all = triggerRepository.findAll(null);
|
||||
|
||||
assertThat(all.size()).isEqualTo(4);
|
||||
|
||||
String namespacePrefix = "io.kestra.another";
|
||||
String namespace = namespacePrefix + ".ns";
|
||||
Trigger trigger = trigger(tenant).namespace(namespace).build();
|
||||
Trigger trigger = trigger().namespace(namespace).build();
|
||||
triggerRepository.save(trigger);
|
||||
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, null, null);
|
||||
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(4);
|
||||
assertThat(find.getFirst().getNamespace()).isEqualTo(namespace);
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, searchedTrigger.getFlowId(), null);
|
||||
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, searchedTrigger.getFlowId(), null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getFlowId()).isEqualTo(searchedTrigger.getFlowId());
|
||||
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, tenant, namespacePrefix, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, null, namespacePrefix, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
|
||||
// Full text search is on namespace, flowId, triggerId, executionId
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), tenant, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), null, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), tenant, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), null, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), tenant, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), null, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), tenant, null, null, null);
|
||||
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), null, null, null, null);
|
||||
assertThat(find.size()).isEqualTo(1);
|
||||
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
|
||||
}
|
||||
@@ -182,17 +178,15 @@ public abstract class AbstractTriggerRepositoryTest {
|
||||
@Test
|
||||
void shouldCountForNullTenant() {
|
||||
// Given
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
triggerRepository.save(Trigger
|
||||
.builder()
|
||||
.tenantId(tenant)
|
||||
.triggerId(IdUtils.create())
|
||||
.flowId(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.build()
|
||||
);
|
||||
// When
|
||||
int count = triggerRepository.count(tenant);
|
||||
int count = triggerRepository.count(null);
|
||||
// Then
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@@ -1,92 +1,88 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
|
||||
class ExecutionFixture {
|
||||
public static Execution EXECUTION_1(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
public static final Execution EXECUTION_1 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", "value"))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", "value"
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
|
||||
public static Execution EXECUTION_2(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
public static final Execution EXECUTION_2 = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
|
||||
public static Execution EXECUTION_TEST(String tenant) {
|
||||
return Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenant)
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
public static final Execution EXECUTION_TEST = Execution.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.flowRevision(1)
|
||||
.state(new State())
|
||||
.inputs(ImmutableMap.of("test", 1))
|
||||
.kind(ExecutionKind.TEST)
|
||||
.taskRunList(Collections.singletonList(
|
||||
TaskRun.builder()
|
||||
.id(IdUtils.create())
|
||||
.namespace("io.kestra.unittest")
|
||||
.flowId("full")
|
||||
.state(new State())
|
||||
.attempts(Collections.singletonList(
|
||||
TaskRunAttempt.builder()
|
||||
.build()
|
||||
))
|
||||
.outputs(Variables.inMemory(ImmutableMap.of(
|
||||
"out", 1
|
||||
)))
|
||||
.build()
|
||||
))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import io.kestra.core.junit.annotations.ExecuteFlow;
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.junit.annotations.LoadFlows;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
@@ -9,35 +13,36 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.plugin.core.flow.*;
|
||||
import io.kestra.plugin.core.flow.EachSequentialTest;
|
||||
import io.kestra.plugin.core.flow.FlowCaseTest;
|
||||
import io.kestra.plugin.core.flow.ForEachItemCaseTest;
|
||||
import io.kestra.plugin.core.flow.PauseTest;
|
||||
import io.kestra.plugin.core.flow.LoopUntilCaseTest;
|
||||
import io.kestra.plugin.core.flow.WorkingDirectoryTest;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
//@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
|
||||
// must be per-class to allow calling once init() which took a lot of time
|
||||
public abstract class AbstractRunnerTest {
|
||||
|
||||
public static final String TENANT_1 = "tenant1";
|
||||
public static final String TENANT_2 = "tenant2";
|
||||
@Inject
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
protected RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
|
||||
protected QueueInterface<LogEntry> logsQueue;
|
||||
|
||||
@Inject
|
||||
protected RestartCaseTest restartCaseTest;
|
||||
private RestartCaseTest restartCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowTriggerCaseTest flowTriggerCaseTest;
|
||||
@@ -49,13 +54,13 @@ public abstract class AbstractRunnerTest {
|
||||
private PluginDefaultsCaseTest pluginDefaultsCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowCaseTest flowCaseTest;
|
||||
private FlowCaseTest flowCaseTest;
|
||||
|
||||
@Inject
|
||||
private WorkingDirectoryTest.Suite workingDirectoryTest;
|
||||
|
||||
@Inject
|
||||
protected PauseTest.Suite pauseTest;
|
||||
private PauseTest.Suite pauseTest;
|
||||
|
||||
@Inject
|
||||
private SkipExecutionCaseTest skipExecutionCaseTest;
|
||||
@@ -67,10 +72,10 @@ public abstract class AbstractRunnerTest {
|
||||
protected LoopUntilCaseTest loopUntilTestCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
private FlowConcurrencyCaseTest flowConcurrencyCaseTest;
|
||||
|
||||
@Inject
|
||||
protected ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
private ScheduleDateCaseTest scheduleDateCaseTest;
|
||||
|
||||
@Inject
|
||||
protected FlowInputOutput flowIO;
|
||||
@@ -79,7 +84,7 @@ public abstract class AbstractRunnerTest {
|
||||
private SLATestCase slaTestCase;
|
||||
|
||||
@Inject
|
||||
protected ChangeStateTestCase changeStateTestCase;
|
||||
private ChangeStateTestCase changeStateTestCase;
|
||||
|
||||
@Inject
|
||||
private AfterExecutionTestCase afterExecutionTestCase;
|
||||
@@ -110,7 +115,7 @@ public abstract class AbstractRunnerTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(8);
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@ExecuteFlow("flows/valids/parallel-nested.yaml")
|
||||
void parallelNested(Execution execution) {
|
||||
assertThat(execution.getTaskRunList()).hasSize(11);
|
||||
@@ -152,27 +157,27 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedThenSuccess();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
void replay() throws Exception {
|
||||
restartCaseTest.replay();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/failed-first.yaml"})
|
||||
void restartMultiple() throws Exception {
|
||||
restartCaseTest.restartMultiple();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows({"flows/valids/restart_always_failed.yaml"})
|
||||
void restartFailedThenFailureWithGlobalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithGlobalErrors();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/restart_local_errors.yaml"})
|
||||
protected void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
void restartFailedThenFailureWithLocalErrors() throws Exception {
|
||||
restartCaseTest.restartFailedThenFailureWithLocalErrors();
|
||||
}
|
||||
|
||||
@@ -194,27 +199,29 @@ public abstract class AbstractRunnerTest {
|
||||
restartCaseTest.restartFailedWithAfterExecution();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-no-inputs.yaml",
|
||||
"flows/valids/trigger-flow-listener.yaml",
|
||||
"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-flow.yaml"}, tenantId = "listener-tenant")
|
||||
"flows/valids/trigger-flow.yaml"})
|
||||
void flowTrigger() throws Exception {
|
||||
flowTriggerCaseTest.trigger("listener-tenant");
|
||||
flowTriggerCaseTest.trigger();
|
||||
}
|
||||
|
||||
@Test // flaky on CI but never fail locally
|
||||
@RetryingTest(5) // flaky on CI but never fail locally
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-pause.yaml",
|
||||
"flows/valids/trigger-flow-with-pause.yaml"})
|
||||
void flowTriggerWithPause() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithPause();
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@Disabled
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"}, tenantId = "trigger-tenant")
|
||||
protected void flowTriggerWithConcurrencyLimit() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit("trigger-tenant");
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
|
||||
"flows/valids/trigger-flow-with-concurrency-limit.yaml"})
|
||||
void flowTriggerWithConcurrencyLimit() throws Exception {
|
||||
flowTriggerCaseTest.triggerWithConcurrencyLimit();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -225,12 +232,12 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.trigger();
|
||||
}
|
||||
|
||||
@Test // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows(value = {"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
|
||||
@LoadFlows({"flows/valids/trigger-flow-listener-namespace-condition.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-c.yaml",
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"}, tenantId = "condition-tenant")
|
||||
"flows/valids/trigger-multiplecondition-flow-d.yaml"})
|
||||
void multipleConditionTriggerFailed() throws Exception {
|
||||
multipleConditionTriggerCaseTest.failed("condition-tenant");
|
||||
multipleConditionTriggerCaseTest.failed();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -242,11 +249,11 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
@LoadFlows({"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-a.yaml",
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"}, tenantId = TENANT_1)
|
||||
"flows/valids/flow-trigger-preconditions-flow-b.yaml"})
|
||||
void flowTriggerPreconditionsMergeOutputs() throws Exception {
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs(TENANT_1);
|
||||
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -255,13 +262,7 @@ public abstract class AbstractRunnerTest {
|
||||
multipleConditionTriggerCaseTest.flowTriggerOnPaused();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-trigger-for-each-item-parent.yaml", "flows/valids/flow-trigger-for-each-item-child.yaml", "flows/valids/flow-trigger-for-each-item-grandchild.yaml"})
|
||||
void forEachItemWithFlowTrigger() throws Exception {
|
||||
multipleConditionTriggerCaseTest.forEachItemWithFlowTrigger();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/each-null.yaml"})
|
||||
void eachWithNull() throws Exception {
|
||||
EachSequentialTest.eachNullTest(runnerUtils, logsQueue);
|
||||
@@ -273,28 +274,28 @@ public abstract class AbstractRunnerTest {
|
||||
pluginDefaultsCaseTest.taskDefaults();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
protected void flowWaitSuccess() throws Exception {
|
||||
void flowWaitSuccess() throws Exception {
|
||||
flowCaseTest.waitSuccess();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_1)
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
void flowWaitFailed() throws Exception {
|
||||
flowCaseTest.waitFailed(TENANT_1);
|
||||
flowCaseTest.waitFailed();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/switch.yaml",
|
||||
@LoadFlows({"flows/valids/switch.yaml",
|
||||
"flows/valids/task-flow.yaml",
|
||||
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_2)
|
||||
"flows/valids/task-flow-inherited-labels.yaml"})
|
||||
public void invalidOutputs() throws Exception {
|
||||
flowCaseTest.invalidOutputs(TENANT_2);
|
||||
flowCaseTest.invalidOutputs();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -304,9 +305,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/working-directory.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/working-directory.yaml"})
|
||||
public void workerFailed() throws Exception {
|
||||
workingDirectoryTest.failed(TENANT_1, runnerUtils);
|
||||
workingDirectoryTest.failed(runnerUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -321,7 +322,7 @@ public abstract class AbstractRunnerTest {
|
||||
workingDirectoryTest.cache(runnerUtils);
|
||||
}
|
||||
|
||||
@Test // flaky on MySQL
|
||||
@RetryingTest(5) // flaky on MySQL
|
||||
@LoadFlows({"flows/valids/pause-test.yaml"})
|
||||
public void pauseRun() throws Exception {
|
||||
pauseTest.run(runnerUtils);
|
||||
@@ -357,44 +358,44 @@ public abstract class AbstractRunnerTest {
|
||||
skipExecutionCaseTest.skipExecution();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item.yaml"})
|
||||
protected void forEachItem() throws Exception {
|
||||
forEachItemCaseTest.forEachItem();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item.yaml"}, tenantId = TENANT_1)
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item.yaml"})
|
||||
protected void forEachItemEmptyItems() throws Exception {
|
||||
forEachItemCaseTest.forEachItemEmptyItems(TENANT_1);
|
||||
forEachItemCaseTest.forEachItemEmptyItems();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow-failed.yaml",
|
||||
"flows/valids/for-each-item-failed.yaml"})
|
||||
protected void forEachItemFailed() throws Exception {
|
||||
forEachItemCaseTest.forEachItemFailed();
|
||||
}
|
||||
|
||||
@Test
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-outputs-subflow.yaml",
|
||||
"flows/valids/for-each-item-outputs.yaml"})
|
||||
protected void forEachItemSubflowOutputs() throws Exception {
|
||||
forEachItemCaseTest.forEachItemWithSubflowOutputs();
|
||||
}
|
||||
|
||||
@Test // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows(value = {"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"}, tenantId = TENANT_1)
|
||||
@RetryingTest(5) // flaky on CI but always pass locally even with 100 iterations
|
||||
@LoadFlows({"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"})
|
||||
void restartForEachItem() throws Exception {
|
||||
forEachItemCaseTest.restartForEachItem(TENANT_1);
|
||||
forEachItemCaseTest.restartForEachItem();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"}, tenantId = TENANT_1)
|
||||
@RetryingTest(5)
|
||||
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
|
||||
"flows/valids/for-each-item-in-if.yaml"})
|
||||
protected void forEachItemInIf() throws Exception {
|
||||
forEachItemCaseTest.forEachItemInIf(TENANT_1);
|
||||
forEachItemCaseTest.forEachItemInIf();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -435,9 +436,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"})
|
||||
protected void flowConcurrencyWithForEachItem() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem(TENANT_1);
|
||||
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -453,15 +454,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"})
|
||||
void flowConcurrencySubflow() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencySubflow(TENANT_1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/flow-concurrency-parallel-subflow-kill.yaml", "flows/valids/flow-concurrency-parallel-subflow-kill-child.yaml", "flows/valids/flow-concurrency-parallel-subflow-kill-grandchild.yaml"})
|
||||
void flowConcurrencyParallelSubflowKill() throws Exception {
|
||||
flowConcurrencyCaseTest.flowConcurrencyParallelSubflowKill();
|
||||
flowConcurrencyCaseTest.flowConcurrencySubflow();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -516,9 +511,9 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/minimal.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/minimal.yaml"})
|
||||
void shouldScheduleOnDate() throws Exception {
|
||||
scheduleDateCaseTest.shouldScheduleOnDate(TENANT_1);
|
||||
scheduleDateCaseTest.shouldScheduleOnDate();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -540,15 +535,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
void executionConditionSLAShouldCancel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldCancel(TENANT_1);
|
||||
slaTestCase.executionConditionSLAShouldCancel();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_2)
|
||||
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
|
||||
void executionConditionSLAShouldLabel() throws Exception {
|
||||
slaTestCase.executionConditionSLAShouldLabel(TENANT_2);
|
||||
slaTestCase.executionConditionSLAShouldLabel();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -568,15 +563,15 @@ public abstract class AbstractRunnerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow(value = "flows/valids/failed-first.yaml", tenantId = TENANT_1)
|
||||
@ExecuteFlow("flows/valids/failed-first.yaml")
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
changeStateTestCase.changeStateShouldEndsInSuccess(execution);
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"}, tenantId = TENANT_2)
|
||||
@LoadFlows({"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"})
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess(TENANT_2);
|
||||
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -3,18 +3,25 @@ package io.kestra.core.runners;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@Singleton
|
||||
public class ChangeStateTestCase {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests";
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@@ -22,7 +29,11 @@ public class ChangeStateTestCase {
|
||||
private ExecutionService executionService;
|
||||
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
private QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
@@ -30,40 +41,73 @@ public class ChangeStateTestCase {
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the last execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
Execution markedAs = executionService.markAs(execution, flow, execution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
Execution lastExecution = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), markedAs);
|
||||
executionQueue.emit(markedAs);
|
||||
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(2);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess(String tenantId) throws Exception {
|
||||
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
|
||||
// await for the subflow execution
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastExecution = new AtomicReference<>();
|
||||
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if ("failed-first".equals(exec.getFlowId()) && exec.getState().getCurrent() == State.Type.FAILED) {
|
||||
lastExecution.set(exec);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// run the parent flow
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-parent-of-failed");
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// assert on the subflow
|
||||
Execution lastExecution = runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.FAILED), tenantId, NAMESPACE, "failed-first");
|
||||
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(lastExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// await for the parent execution
|
||||
CountDownLatch parentLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> lastParentExecution = new AtomicReference<>();
|
||||
receivedExecutions = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution exec = either.getLeft();
|
||||
if (execution.getId().equals(exec.getId()) && exec.getState().isTerminated()) {
|
||||
lastParentExecution.set(exec);
|
||||
parentLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// restart the subflow
|
||||
Flow flow = flowRepository.findByExecution(lastExecution);
|
||||
Execution markedAs = executionService.markAs(lastExecution, flow, lastExecution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().isTerminated(), markedAs);
|
||||
|
||||
//We wait for the subflow execution to pass from failed to success
|
||||
Execution lastParentExecution = runnerUtils.awaitFlowExecution(e ->
|
||||
e.getTaskRunList().getFirst().getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "subflow-parent-of-failed");
|
||||
Flow flow = flowRepository.findByExecution(lastExecution.get());
|
||||
Execution markedAs = executionService.markAs(lastExecution.get(), flow, lastExecution.get().getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
|
||||
executionQueue.emit(markedAs);
|
||||
|
||||
// assert for the parent flow
|
||||
assertThat(lastParentExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(parentLatch.await(10, TimeUnit.SECONDS)).isTrue();
|
||||
receivedExecutions.blockLast();
|
||||
assertThat(lastParentExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
|
||||
assertThat(lastParentExecution.get().getTaskRunList()).hasSize(1);
|
||||
assertThat(lastParentExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
public class EmptyVariablesTest {
|
||||
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
private RunnerUtils runnerUtils;
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
|
||||
|
||||
@@ -40,10 +40,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
@Slf4j
|
||||
@KestraTest(startRunner = true)
|
||||
class ExecutionServiceTest {
|
||||
|
||||
public static final String TENANT_1 = "tenant1";
|
||||
public static final String TENANT_2 = "tenant2";
|
||||
public static final String TENANT_3 = "tenant3";
|
||||
@Inject
|
||||
ExecutionService executionService;
|
||||
|
||||
@@ -57,7 +53,7 @@ class ExecutionServiceTest {
|
||||
LogRepositoryInterface logRepository;
|
||||
|
||||
@Inject
|
||||
TestRunnerUtils runnerUtils;
|
||||
RunnerUtils runnerUtils;
|
||||
|
||||
@Test
|
||||
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
|
||||
@@ -72,20 +68,20 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getTaskRunList()).hasSize(3);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(5);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getId()).isEqualTo(execution.getId());
|
||||
assertThat(restart.getTaskRunList().get(2).getId()).isEqualTo(execution.getTaskRunList().get(2).getId());
|
||||
assertThat(restart.getLabels()).contains(new Label(Label.RESTARTED, "true"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/restart_last_failed.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
|
||||
void restartSimpleRevision() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart_last_failed");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart_last_failed");
|
||||
assertThat(execution.getTaskRunList()).hasSize(3);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
FlowWithSource flow = flowRepository.findByIdWithSource(TENANT_1, "io.kestra.tests", "restart_last_failed").orElseThrow();
|
||||
FlowWithSource flow = flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests", "restart_last_failed").orElseThrow();
|
||||
flowRepository.update(
|
||||
GenericFlow.of(flow),
|
||||
flow.updateTask(
|
||||
@@ -105,7 +101,7 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getTaskRunList()).hasSize(3);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(5);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getId()).isNotEqualTo(execution.getId());
|
||||
assertThat(restart.getTaskRunList().get(2).getId()).isNotEqualTo(execution.getTaskRunList().get(2).getId());
|
||||
assertThat(restart.getLabels()).contains(new Label(Label.RESTARTED, "true"));
|
||||
@@ -128,9 +124,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@RetryingTest(5)
|
||||
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
void restartFlowable2() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
Execution restart = executionService.restart(execution, null);
|
||||
@@ -181,9 +177,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/logs.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/logs.yaml"})
|
||||
void replaySimple() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "logs");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "logs");
|
||||
assertThat(execution.getTaskRunList()).hasSize(5);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
@@ -193,16 +189,16 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getTaskRunList()).hasSize(2);
|
||||
assertThat(restart.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restart.getTaskRunList().get(1).getState().getHistories()).hasSize(5);
|
||||
assertThat(restart.getTaskRunList().get(1).getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getId()).isNotEqualTo(execution.getId());
|
||||
assertThat(restart.getTaskRunList().get(1).getId()).isNotEqualTo(execution.getTaskRunList().get(1).getId());
|
||||
assertThat(restart.getLabels()).contains(new Label(Label.REPLAY, "true"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_2)
|
||||
@LoadFlows({"flows/valids/restart-each.yaml"})
|
||||
void replayFlowable() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(TENANT_2, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
|
||||
assertThat(execution.getTaskRunList()).hasSize(20);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
@@ -238,7 +234,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_2)
|
||||
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
|
||||
void replayEachSeq(Execution execution) throws Exception {
|
||||
assertThat(execution.getTaskRunList()).hasSize(23);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
@@ -257,7 +253,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_1)
|
||||
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
|
||||
void replayEachSeq2(Execution execution) throws Exception {
|
||||
assertThat(execution.getTaskRunList()).hasSize(23);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
@@ -288,7 +284,7 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.getTaskRunList()).hasSize(3);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(5);
|
||||
assertThat(restart.getTaskRunList().get(2).getState().getHistories()).hasSize(4);
|
||||
|
||||
assertThat(restart.getId()).isNotEqualTo(execution.getId());
|
||||
assertThat(restart.getTaskRunList().get(1).getId()).isNotEqualTo(execution.getTaskRunList().get(1).getId());
|
||||
@@ -316,9 +312,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/each-parallel-nested.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/each-parallel-nested.yaml"})
|
||||
void markAsEachPara() throws Exception {
|
||||
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "each-parallel-nested");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "each-parallel-nested");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(11);
|
||||
@@ -343,7 +339,7 @@ class ExecutionServiceTest {
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("1_each", List.of()).getState().getCurrent()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("2-1_seq", List.of("value 1")).getState().getCurrent()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("2-1-2_t2", List.of("value 1")).getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("2-1-2_t2", List.of("value 1")).getState().getHistories()).hasSize(5);
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("2-1-2_t2", List.of("value 1")).getState().getHistories()).hasSize(4);
|
||||
assertThat(restart.findTaskRunByTaskIdAndValue("2-1-2_t2", List.of("value 1")).getAttempts().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
}
|
||||
|
||||
@@ -368,9 +364,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/pause-test.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/pause-test.yaml"})
|
||||
void resumePausedToKilling() throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause-test");
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause-test");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
@@ -383,7 +379,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_2)
|
||||
@ExecuteFlow("flows/valids/logs.yaml")
|
||||
void deleteExecution(Execution execution) throws IOException, TimeoutException {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
@@ -395,7 +391,7 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_3)
|
||||
@ExecuteFlow("flows/valids/logs.yaml")
|
||||
void deleteExecutionKeepLogs(Execution execution) throws IOException, TimeoutException {
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
@@ -435,9 +431,9 @@ class ExecutionServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/pause_no_tasks.yaml"}, tenantId = TENANT_1)
|
||||
@LoadFlows({"flows/valids/pause_no_tasks.yaml"})
|
||||
void killToState() throws Exception {
|
||||
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause_no_tasks");
|
||||
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause_no_tasks");
|
||||
Flow flow = flowRepository.findByExecution(execution);
|
||||
|
||||
Execution killed = executionService.kill(execution, flow, Optional.of(State.Type.CANCELLED));
|
||||
|
||||
@@ -9,7 +9,6 @@ import io.micronaut.context.annotation.Property;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
@@ -19,14 +18,11 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest(rebuildContext = true)
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class FilesServiceTest {
|
||||
@Inject
|
||||
private TestRunContextFactory runContextFactory;
|
||||
@@ -101,35 +97,6 @@ class FilesServiceTest {
|
||||
assertThat(outputs.size()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testOutputFilesWithSpecialCharacters(@TempDir Path tempDir) throws Exception {
|
||||
var runContext = runContextFactory.of();
|
||||
|
||||
Path fileWithSpace = tempDir.resolve("with space.txt");
|
||||
Path fileWithUnicode = tempDir.resolve("สวัสดี.txt");
|
||||
|
||||
Files.writeString(fileWithSpace, "content");
|
||||
Files.writeString(fileWithUnicode, "content");
|
||||
|
||||
Path targetFileWithSpace = runContext.workingDir().path().resolve("with space.txt");
|
||||
Path targetFileWithUnicode = runContext.workingDir().path().resolve("สวัสดี.txt");
|
||||
|
||||
Files.copy(fileWithSpace, targetFileWithSpace);
|
||||
Files.copy(fileWithUnicode, targetFileWithUnicode);
|
||||
|
||||
Map<String, URI> outputFiles = FilesService.outputFiles(
|
||||
runContext,
|
||||
List.of("with space.txt", "สวัสดี.txt")
|
||||
);
|
||||
|
||||
assertThat(outputFiles).hasSize(2);
|
||||
assertThat(outputFiles).containsKey("with space.txt");
|
||||
assertThat(outputFiles).containsKey("สวัสดี.txt");
|
||||
|
||||
assertThat(runContext.storage().getFile(outputFiles.get("with space.txt"))).isNotNull();
|
||||
assertThat(runContext.storage().getFile(outputFiles.get("สวัสดี.txt"))).isNotNull();
|
||||
}
|
||||
|
||||
private URI createFile() throws IOException {
|
||||
File tempFile = File.createTempFile("file", ".txt");
|
||||
Files.write(tempFile.toPath(), "Hello World".getBytes());
|
||||
|
||||
@@ -5,11 +5,11 @@ import io.kestra.core.models.executions.ExecutionKilled;
|
||||
import io.kestra.core.models.executions.ExecutionKilledExecution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.History;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.reporter.model.Count;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.ExecutionService;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
@@ -18,6 +18,7 @@ import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
@@ -27,21 +28,24 @@ import java.net.URISyntaxException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class FlowConcurrencyCaseTest {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests";
|
||||
@Inject
|
||||
private StorageInterface storageInterface;
|
||||
|
||||
@Inject
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
protected RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private FlowInputOutput flowIO;
|
||||
@@ -49,6 +53,10 @@ public class FlowConcurrencyCaseTest {
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
private ExecutionService executionService;
|
||||
|
||||
@@ -56,185 +64,393 @@ public class FlowConcurrencyCaseTest {
|
||||
@Named(QueueFactoryInterface.KILL_NAMED)
|
||||
protected QueueInterface<ExecutionKilled> killQueue;
|
||||
|
||||
public void flowConcurrencyCancel() throws TimeoutException, QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel");
|
||||
public void flowConcurrencyCancel() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel");
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
|
||||
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
}
|
||||
|
||||
public void flowConcurrencyFail() throws TimeoutException, QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail");
|
||||
public void flowConcurrencyFail() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail");
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueue() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
|
||||
public void flowConcurrencyQueue() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", Optional.empty())
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
executionQueue.emit(execution2);
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueuePause() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause");
|
||||
public void flowConcurrencyQueuePause() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<String> firstExecutionId = new AtomicReference<>();
|
||||
var firstExecutionResult = new AtomicReference<Execution>();
|
||||
var secondExecutionResult = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch firstExecutionLatch = new CountDownLatch(1);
|
||||
CountDownLatch secondExecutionLatch = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (!"flow-concurrency-queue-pause".equals(e.getLeft().getFlowId())){
|
||||
return;
|
||||
}
|
||||
String currentId = e.getLeft().getId();
|
||||
Type currentState = e.getLeft().getState().getCurrent();
|
||||
if (firstExecutionId.get() == null) {
|
||||
firstExecutionId.set(currentId);
|
||||
}
|
||||
|
||||
if (currentId.equals(firstExecutionId.get())) {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
firstExecutionResult.set(e.getLeft());
|
||||
firstExecutionLatch.countDown();
|
||||
}
|
||||
} else {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
secondExecutionResult.set(e.getLeft());
|
||||
secondExecutionLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause");
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause", Optional.empty())
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
executionQueue.emit(execution2);
|
||||
|
||||
assertThat(execution1.getState().isPaused()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertTrue(firstExecutionLatch.await(10, TimeUnit.SECONDS));
|
||||
assertTrue(secondExecutionLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyCancelPause() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause");
|
||||
public void flowConcurrencyCancelPause() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<String> firstExecutionId = new AtomicReference<>();
|
||||
var firstExecutionResult = new AtomicReference<Execution>();
|
||||
var secondExecutionResult = new AtomicReference<Execution>();
|
||||
CountDownLatch firstExecLatch = new CountDownLatch(1);
|
||||
CountDownLatch secondExecLatch = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (!"flow-concurrency-cancel-pause".equals(e.getLeft().getFlowId())){
|
||||
return;
|
||||
}
|
||||
String currentId = e.getLeft().getId();
|
||||
Type currentState = e.getLeft().getState().getCurrent();
|
||||
if (firstExecutionId.get() == null) {
|
||||
firstExecutionId.set(currentId);
|
||||
}
|
||||
if (currentId.equals(firstExecutionId.get())) {
|
||||
if (currentState == State.Type.SUCCESS) {
|
||||
firstExecutionResult.set(e.getLeft());
|
||||
firstExecLatch.countDown();
|
||||
}
|
||||
} else {
|
||||
if (currentState == State.Type.CANCELLED) {
|
||||
secondExecutionResult.set(e.getLeft());
|
||||
secondExecLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause");
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause", Optional.empty())
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.CANCELLED), execution2);
|
||||
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
executionQueue.emit(execution2);
|
||||
|
||||
assertThat(execution1.getState().isPaused()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
|
||||
assertTrue(firstExecLatch.await(10, TimeUnit.SECONDS));
|
||||
assertTrue(secondExecLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
|
||||
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
|
||||
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
|
||||
}
|
||||
|
||||
public void flowConcurrencyWithForEachItem(String tenantId) throws QueueException, URISyntaxException, IOException {
|
||||
URI file = storageUpload(tenantId);
|
||||
public void flowConcurrencyWithForEachItem() throws TimeoutException, QueueException, InterruptedException, URISyntaxException, IOException {
|
||||
URI file = storageUpload();
|
||||
Map<String, Object> inputs = Map.of("file", file.toString(), "batch", 4);
|
||||
Execution forEachItem = runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-for-each-item", null,
|
||||
Execution forEachItem = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-for-each-item", null,
|
||||
(flow, execution1) -> flowIO.readExecutionInputs(flow, execution1, inputs), Duration.ofSeconds(5));
|
||||
assertThat(forEachItem.getState().getCurrent()).isEqualTo(Type.RUNNING);
|
||||
|
||||
Set<String> executionIds = new HashSet<>();
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if ("flow-concurrency-queue".equals(e.getLeft().getFlowId()) && e.getLeft().getState().isRunning()) {
|
||||
executionIds.add(e.getLeft().getId());
|
||||
}
|
||||
});
|
||||
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getState().isTerminated(),forEachItem);
|
||||
// wait a little to be sure there are not too many executions started
|
||||
Thread.sleep(500);
|
||||
|
||||
assertThat(executionIds).hasSize(1);
|
||||
receive.blockLast();
|
||||
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getId().equals(forEachItem.getId()) && e.getState().isTerminated(), () -> {}, Duration.ofSeconds(10));
|
||||
assertThat(terminated.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
|
||||
List<Execution> executions = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-queue");
|
||||
|
||||
assertThat(executions).extracting(e -> e.getState().getCurrent()).containsOnly(Type.SUCCESS);
|
||||
assertThat(executions.stream()
|
||||
.map(e -> e.getState().getHistories())
|
||||
.flatMap(List::stream)
|
||||
.map(History::getState)
|
||||
.toList()).contains(Type.QUEUED);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueueRestarted() throws Exception {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE,
|
||||
"flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-fail", Optional.empty())
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.RUNNING), execution2);
|
||||
executionQueue.emit(execution2);
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(2);
|
||||
AtomicReference<Execution> failedExecution = new AtomicReference<>();
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
|
||||
failedExecution.set(e.getLeft());
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertThat(failedExecution.get()).isNotNull();
|
||||
// here the first fail and the second is now running.
|
||||
// we restart the first one, it should be queued then fail again.
|
||||
Execution failedExecution = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution1);
|
||||
Execution restarted = executionService.restart(failedExecution, null);
|
||||
Execution executionResult1 = runnerUtils.restartExecution(e -> e.getState().getCurrent().equals(Type.FAILED), restarted);
|
||||
Execution executionResult2 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution2);
|
||||
Execution restarted = executionService.restart(failedExecution.get(), null);
|
||||
executionQueue.emit(restarted);
|
||||
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
// it should have been queued after restarted
|
||||
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
|
||||
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
|
||||
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(Type.FAILED);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencyQueueAfterExecution() throws QueueException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
|
||||
public void flowConcurrencyQueueAfterExecution() throws TimeoutException, QueueException, InterruptedException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
|
||||
Flow flow = flowRepository
|
||||
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", Optional.empty())
|
||||
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", Optional.empty())
|
||||
.orElseThrow();
|
||||
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
|
||||
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
|
||||
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
|
||||
executionQueue.emit(execution2);
|
||||
|
||||
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
|
||||
|
||||
var executionResult1 = new AtomicReference<Execution>();
|
||||
var executionResult2 = new AtomicReference<Execution>();
|
||||
|
||||
CountDownLatch latch1 = new CountDownLatch(1);
|
||||
CountDownLatch latch2 = new CountDownLatch(1);
|
||||
CountDownLatch latch3 = new CountDownLatch(1);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getId().equals(execution1.getId())) {
|
||||
executionResult1.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch1.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
if (e.getLeft().getId().equals(execution2.getId())) {
|
||||
executionResult2.set(e.getLeft());
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
|
||||
latch2.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
latch3.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
assertTrue(latch1.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch2.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(latch3.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
|
||||
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public void flowConcurrencySubflow(String tenantId) throws TimeoutException, QueueException {
|
||||
runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-subflow", null, null, Duration.ofSeconds(30));
|
||||
runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
|
||||
public void flowConcurrencySubflow() throws TimeoutException, QueueException, InterruptedException {
|
||||
CountDownLatch successLatch = new CountDownLatch(1);
|
||||
CountDownLatch canceledLatch = new CountDownLatch(1);
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getFlowId().equals("flow-concurrency-cancel")) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
successLatch.countDown();
|
||||
}
|
||||
if (e.getLeft().getState().getCurrent() == Type.CANCELLED) {
|
||||
canceledLatch.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
List<Execution> subFlowExecs = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-cancel");
|
||||
assertThat(subFlowExecs).extracting(e -> e.getState().getCurrent()).containsExactlyInAnyOrder(Type.SUCCESS, Type.CANCELLED);
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-subflow", null, null, Duration.ofSeconds(30));
|
||||
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-subflow");
|
||||
|
||||
assertThat(execution1.getState().isRunning()).isTrue();
|
||||
assertThat(execution2.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
|
||||
// assert we have one canceled subflow and one in success
|
||||
assertTrue(canceledLatch.await(1, TimeUnit.MINUTES));
|
||||
assertTrue(successLatch.await(1, TimeUnit.MINUTES));
|
||||
receive.blockLast();
|
||||
|
||||
// run another execution to be sure that everything work (purge is correctly done)
|
||||
Execution execution3 = runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
|
||||
CountDownLatch newSuccessLatch = new CountDownLatch(1);
|
||||
Flux<Execution> secondReceive = TestsUtils.receive(executionQueue, e -> {
|
||||
if (e.getLeft().getFlowId().equals("flow-concurrency-cancel")) {
|
||||
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
|
||||
newSuccessLatch.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
|
||||
});
|
||||
Execution execution3 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-subflow");
|
||||
assertThat(execution3.getState().getCurrent()).isEqualTo(Type.SUCCESS);
|
||||
runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "flow-concurrency-cancel");
|
||||
|
||||
// assert we have two successful subflow
|
||||
assertTrue(newSuccessLatch.await(1, TimeUnit.MINUTES));
|
||||
secondReceive.blockLast();
|
||||
}
|
||||
|
||||
public void flowConcurrencyParallelSubflowKill() throws QueueException {
|
||||
Execution parent = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-parallel-subflow-kill", null, null, Duration.ofSeconds(30));
|
||||
Execution queued = runnerUtils.awaitFlowExecution(e -> e.getState().isQueued(), MAIN_TENANT, NAMESPACE, "flow-concurrency-parallel-subflow-kill-child");
|
||||
|
||||
// Kill the parent
|
||||
killQueue.emit(ExecutionKilledExecution
|
||||
.builder()
|
||||
.state(ExecutionKilled.State.REQUESTED)
|
||||
.executionId(parent.getId())
|
||||
.isOnKillCascade(true)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.build()
|
||||
);
|
||||
|
||||
Execution terminated = runnerUtils.awaitExecution(e -> e.getState().isTerminated(), queued);
|
||||
assertThat(terminated.getState().getCurrent()).isEqualTo(State.Type.KILLED);
|
||||
assertThat(terminated.getState().getHistories().stream().noneMatch(h -> h.getState() == Type.RUNNING)).isTrue();
|
||||
assertThat(terminated.getTaskRunList()).isNull();
|
||||
}
|
||||
|
||||
private URI storageUpload(String tenantId) throws URISyntaxException, IOException {
|
||||
private URI storageUpload() throws URISyntaxException, IOException {
|
||||
File tempFile = File.createTempFile("file", ".txt");
|
||||
|
||||
Files.write(tempFile.toPath(), content());
|
||||
|
||||
return storageInterface.put(
|
||||
tenantId,
|
||||
MAIN_TENANT,
|
||||
null,
|
||||
new URI("/file/storage/file.txt"),
|
||||
new FileInputStream(tempFile)
|
||||
|
||||
@@ -4,22 +4,19 @@ import io.kestra.core.models.flows.FlowWithSource;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.junit.annotations.KestraTest;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import lombok.SneakyThrows;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.services.FlowListenersInterface;
|
||||
import io.kestra.plugin.core.debug.Return;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import jakarta.inject.Inject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@@ -27,11 +24,11 @@ abstract public class FlowListenersTest {
|
||||
@Inject
|
||||
protected FlowRepositoryInterface flowRepository;
|
||||
|
||||
protected static FlowWithSource create(String tenantId, String flowId, String taskId) {
|
||||
protected static FlowWithSource create(String flowId, String taskId) {
|
||||
FlowWithSource flow = FlowWithSource.builder()
|
||||
.id(flowId)
|
||||
.namespace("io.kestra.unittest")
|
||||
.tenantId(tenantId)
|
||||
.tenantId(MAIN_TENANT)
|
||||
.revision(1)
|
||||
.tasks(Collections.singletonList(Return.builder()
|
||||
.id(taskId)
|
||||
@@ -42,65 +39,88 @@ abstract public class FlowListenersTest {
|
||||
return flow.toBuilder().source(flow.sourceOrGenerateIfNull()).build();
|
||||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(FlowListenersTest.class);
|
||||
|
||||
public void suite(FlowListenersInterface flowListenersService) throws TimeoutException {
|
||||
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
|
||||
public void suite(FlowListenersInterface flowListenersService) {
|
||||
flowListenersService.run();
|
||||
|
||||
AtomicInteger count = new AtomicInteger();
|
||||
var ref = new Ref();
|
||||
|
||||
flowListenersService.listen(flows -> count.set(getFlowsForTenant(flowListenersService, tenant).size()));
|
||||
flowListenersService.listen(flows -> {
|
||||
count.set(flows.size());
|
||||
ref.countDownLatch.countDown();
|
||||
});
|
||||
|
||||
// initial state
|
||||
LOG.info("-----------> wait for zero");
|
||||
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isZero();
|
||||
assertThat(flowListenersService.flows().size()).isZero();
|
||||
});
|
||||
|
||||
// resend on startup done for kafka
|
||||
LOG.info("-----------> wait for zero kafka");
|
||||
if (flowListenersService.getClass().getName().equals("io.kestra.ee.runner.kafka.KafkaFlowListeners")) {
|
||||
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isZero();
|
||||
assertThat(flowListenersService.flows().size()).isZero();
|
||||
});
|
||||
}
|
||||
|
||||
// create first
|
||||
LOG.info("-----------> create fist flow");
|
||||
FlowWithSource first = create(tenant, "first_" + IdUtils.create(), "test");
|
||||
FlowWithSource firstUpdated = create(tenant, first.getId(), "test2");
|
||||
FlowWithSource first = create("first_" + IdUtils.create(), "test");
|
||||
FlowWithSource firstUpdated = create(first.getId(), "test2");
|
||||
|
||||
|
||||
flowRepository.create(GenericFlow.of(first));
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
});
|
||||
|
||||
// create the same id than first, no additional flows
|
||||
first = flowRepository.update(GenericFlow.of(firstUpdated), first);
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
//assertThat(flowListenersService.flows().getFirst().getFirst().getId(), is("test2"));
|
||||
});
|
||||
|
||||
FlowWithSource second = create(tenant, "second_" + IdUtils.create(), "test");
|
||||
FlowWithSource second = create("second_" + IdUtils.create(), "test");
|
||||
// create a new one
|
||||
flowRepository.create(GenericFlow.of(second));
|
||||
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(2);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(2);
|
||||
});
|
||||
|
||||
// delete first
|
||||
FlowWithSource deleted = flowRepository.delete(first);
|
||||
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(1);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(1);
|
||||
});
|
||||
|
||||
// restore must works
|
||||
flowRepository.create(GenericFlow.of(first));
|
||||
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
|
||||
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(2);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(2);
|
||||
});
|
||||
|
||||
FlowWithSource withTenant = first.toBuilder().tenantId("some-tenant").build();
|
||||
flowRepository.create(GenericFlow.of(withTenant));
|
||||
wait(ref, () -> {
|
||||
assertThat(count.get()).isEqualTo(3);
|
||||
assertThat(flowListenersService.flows().size()).isEqualTo(3);
|
||||
});
|
||||
}
|
||||
|
||||
public List<FlowWithSource> getFlowsForTenant(FlowListenersInterface flowListenersService, String tenantId){
|
||||
return flowListenersService.flows().stream()
|
||||
.filter(f -> tenantId.equals(f.getTenantId()))
|
||||
.toList();
|
||||
public static class Ref {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private void wait(Ref ref, Runnable run) {
|
||||
ref.countDownLatch.await(60, TimeUnit.SECONDS);
|
||||
run.run();
|
||||
ref.countDownLatch = new CountDownLatch(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,61 +2,82 @@ package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class FlowTriggerCaseTest {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests.trigger";
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
protected RunnerUtils runnerUtils;
|
||||
|
||||
public void trigger(String tenantId) throws InterruptedException, TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "trigger-flow");
|
||||
public void trigger() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(3);
|
||||
AtomicReference<Execution> flowListener = new AtomicReference<>();
|
||||
AtomicReference<Execution> flowListenerNoInput = new AtomicReference<>();
|
||||
AtomicReference<Execution> flowListenerNamespace = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
if (flowListenerNoInput.get() == null && execution.getFlowId().equals("trigger-flow-listener-no-inputs")) {
|
||||
flowListenerNoInput.set(execution);
|
||||
countDownLatch.countDown();
|
||||
} else if (flowListener.get() == null && execution.getFlowId().equals("trigger-flow-listener")) {
|
||||
flowListener.set(execution);
|
||||
countDownLatch.countDown();
|
||||
} else if (flowListenerNamespace.get() == null && execution.getFlowId().equals("trigger-flow-listener-namespace-condition")) {
|
||||
flowListenerNamespace.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger", "trigger-flow");
|
||||
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
Execution flowListenerNoInput = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener-no-inputs");
|
||||
Execution flowListener = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener");
|
||||
Execution flowListenerNamespace = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
|
||||
"trigger-flow-listener-namespace-condition");
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(flowListener.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(flowListener.get().getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(flowListener.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
|
||||
assertThat(flowListener.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListener.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(flowListener.getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
|
||||
assertThat(flowListener.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListener.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
assertThat(flowListener.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
assertThat(flowListenerNoInput.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
assertThat(flowListenerNoInput.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(flowListenerNoInput.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
assertThat(flowListenerNoInput.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
|
||||
assertThat(flowListenerNoInput.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(flowListenerNamespace.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNamespace.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
|
||||
assertThat(flowListenerNamespace.get().getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
// it will be triggered for 'trigger-flow' or any of the 'trigger-flow-listener*', so we only assert that it's one of them
|
||||
assertThat(flowListenerNamespace.getTrigger().getVariables().get("flowId"))
|
||||
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("flowId"))
|
||||
.satisfiesAnyOf(
|
||||
arg -> assertThat(arg).isEqualTo("trigger-flow"),
|
||||
arg -> assertThat(arg).isEqualTo("trigger-flow-listener-no-inputs"),
|
||||
@@ -64,43 +85,56 @@ public class FlowTriggerCaseTest {
|
||||
);
|
||||
}
|
||||
|
||||
public void triggerWithPause() throws TimeoutException, QueueException {
|
||||
public void triggerWithPause() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(4);
|
||||
List<Execution> flowListeners = new ArrayList<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-pause")) {
|
||||
flowListeners.add(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.pause", "trigger-flow-with-pause");
|
||||
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(3);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
|
||||
4,
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests.trigger.pause",
|
||||
"trigger-flow-listener-with-pause");
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(triggeredExec.size()).isEqualTo(4);
|
||||
List<Execution> sortedExecs = triggeredExec.stream()
|
||||
.sorted(Comparator.comparing(e -> e.getState().getEndDate().orElse(Instant.now())))
|
||||
.toList();
|
||||
assertThat(sortedExecs.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(sortedExecs.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
|
||||
assertThat(sortedExecs.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(sortedExecs.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
|
||||
assertThat(flowListeners.size()).isEqualTo(4);
|
||||
assertThat(flowListeners.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(flowListeners.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
|
||||
assertThat(flowListeners.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
|
||||
assertThat(flowListeners.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
|
||||
}
|
||||
|
||||
public void triggerWithConcurrencyLimit(String tenantId) throws QueueException, TimeoutException {
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
Execution execution2 = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
public void triggerWithConcurrencyLimit() throws QueueException, TimeoutException, InterruptedException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(5);
|
||||
List<Execution> flowListeners = new ArrayList<>();
|
||||
|
||||
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
|
||||
5,
|
||||
tenantId,
|
||||
"io.kestra.tests.trigger.concurrency",
|
||||
"trigger-flow-listener-with-concurrency-limit");
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-concurrency-limit")) {
|
||||
flowListeners.add(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
assertThat(triggeredExec.size()).isEqualTo(5);
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
Execution execution2 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
|
||||
|
||||
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
|
||||
assertThat(flowListeners.size()).isEqualTo(5);
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,9 +17,6 @@ import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -39,7 +36,6 @@ import java.util.concurrent.TimeoutException;
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@KestraTest(startRunner = true)
|
||||
public class InputsTest {
|
||||
@@ -48,7 +44,7 @@ public class InputsTest {
|
||||
private QueueInterface<LogEntry> logQueue;
|
||||
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
public static Map<String, Object> inputs = ImmutableMap.<String, Object>builder()
|
||||
.put("string", "myString")
|
||||
@@ -94,8 +90,8 @@ public class InputsTest {
|
||||
@Inject
|
||||
private FlowInputOutput flowInputOutput;
|
||||
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map, String tenantId) {
|
||||
return typedInputs(map, flowRepository.findById(tenantId, "io.kestra.tests", "inputs").get());
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map) {
|
||||
return typedInputs(map, flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "inputs").get());
|
||||
}
|
||||
|
||||
private Map<String, Object> typedInputs(Map<String, Object> map, Flow flow) {
|
||||
@@ -104,7 +100,7 @@ public class InputsTest {
|
||||
Execution.builder()
|
||||
.id("test")
|
||||
.namespace(flow.getNamespace())
|
||||
.tenantId(flow.getTenantId())
|
||||
.tenantId(MAIN_TENANT)
|
||||
.flowRevision(1)
|
||||
.flowId(flow.getId())
|
||||
.build(),
|
||||
@@ -117,25 +113,25 @@ public class InputsTest {
|
||||
void missingRequired() {
|
||||
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
|
||||
inputs.put("string", null);
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs, MAIN_TENANT));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `string`, missing required input, but received `null`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void nonRequiredNoDefaultNoValueIsNull() {
|
||||
HashMap<String, Object> inputsWithMissingOptionalInput = new HashMap<>(inputs);
|
||||
inputsWithMissingOptionalInput.remove("bool");
|
||||
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").containsKey("bool")).isTrue();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").get("bool")).isNull();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput).containsKey("bool")).isTrue();
|
||||
assertThat(typedInputs(inputsWithMissingOptionalInput).get("bool")).isNull();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant1")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void allValidInputs() throws URISyntaxException, IOException {
|
||||
Map<String, Object> typeds = typedInputs(inputs, "tenant1");
|
||||
Map<String, Object> typeds = typedInputs(inputs);
|
||||
|
||||
assertThat(typeds.get("string")).isEqualTo("myString");
|
||||
assertThat(typeds.get("int")).isEqualTo(42);
|
||||
@@ -147,7 +143,7 @@ public class InputsTest {
|
||||
assertThat(typeds.get("time")).isEqualTo(LocalTime.parse("18:27:49"));
|
||||
assertThat(typeds.get("duration")).isEqualTo(Duration.parse("PT5M6S"));
|
||||
assertThat((URI) typeds.get("file")).isEqualTo(new URI("kestra:///io/kestra/tests/inputs/executions/test/inputs/file/application-test.yml"));
|
||||
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get("tenant1", null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
|
||||
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get(MAIN_TENANT, null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
|
||||
assertThat(typeds.get("json")).isEqualTo(Map.of("a", "b"));
|
||||
assertThat(typeds.get("uri")).isEqualTo("https://www.google.com");
|
||||
assertThat(((Map<String, Object>) typeds.get("nested")).get("string")).isEqualTo("a string");
|
||||
@@ -170,9 +166,9 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant2")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void allValidTypedInputs() {
|
||||
Map<String, Object> typeds = typedInputs(inputs, "tenant2");
|
||||
Map<String, Object> typeds = typedInputs(inputs);
|
||||
typeds.put("int", 42);
|
||||
typeds.put("float", 42.42F);
|
||||
typeds.put("bool", false);
|
||||
@@ -185,10 +181,10 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant3")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputFlow() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(
|
||||
"tenant3",
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -205,165 +201,165 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant4")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedStringBadValue() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("validatedString", "foo");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant4"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedString`, it must match the pattern");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant5")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedIntegerBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedInt", "9");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant5"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be more than `10`, but received `9`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedInt", "21");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant5"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be less than `20`, but received `21`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant6")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedDateBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDate", "2022-01-01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant6"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be after `2023-01-01`, but received `2022-01-01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDate", "2024-01-01");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant6"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be before `2023-12-31`, but received `2024-01-01`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant7")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedDateTimeBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDateTime", "2022-01-01T00:00:00Z");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant7"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be after `2023-01-01T00:00:00Z`, but received `2022-01-01T00:00:00Z`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDateTime", "2024-01-01T00:00:00Z");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant7"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be before `2023-12-31T23:59:59Z`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant8")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedDurationBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedDuration", "PT1S");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant8"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be more than `PT10S`, but received `PT1S`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedDuration", "PT30S");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant8"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be less than `PT20S`, but received `PT30S`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant9")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedFloatBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedFloat", "0.01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant9"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be more than `0.1`, but received `0.01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedFloat", "1.01");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant9"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be less than `0.5`, but received `1.01`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant10")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputValidatedTimeBadValue() {
|
||||
HashMap<String, Object> mapMin = new HashMap<>(inputs);
|
||||
mapMin.put("validatedTime", "00:00:01");
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant10"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be after `01:00`, but received `00:00:01`");
|
||||
|
||||
HashMap<String, Object> mapMax = new HashMap<>(inputs);
|
||||
mapMax.put("validatedTime", "14:00:00");
|
||||
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant10"));
|
||||
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be before `11:59:59`, but received `14:00:00`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant11")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("uri", "http:/bla");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant11"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `uri`, Expected `URI` but received `http:/bla`, but received `http:/bla`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant12")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputEnumFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("enum", "INVALID");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant12"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
|
||||
assertThat(e.getMessage()).isEqualTo("enum: Invalid input for `enum`, it must match the values `[ENUM_VALUE, OTHER_ONE]`, but received `INVALID`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant13")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputArrayFailed() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("array", "[\"s1\", \"s2\"]");
|
||||
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant13"));
|
||||
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
|
||||
|
||||
assertThat(e.getMessage()).contains("Invalid input for `array`, Unable to parse array element as `INT` on `s1`, but received `[\"s1\", \"s2\"]`");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant14")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputEmptyJson() {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("json", "{}");
|
||||
|
||||
Map<String, Object> typeds = typedInputs(map, "tenant14");
|
||||
Map<String, Object> typeds = typedInputs(map);
|
||||
|
||||
assertThat(typeds.get("json")).isInstanceOf(Map.class);
|
||||
assertThat(((Map<?, ?>) typeds.get("json")).size()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant15")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void inputEmptyJsonFlow() throws TimeoutException, QueueException {
|
||||
HashMap<String, Object> map = new HashMap<>(inputs);
|
||||
map.put("json", "{}");
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
"tenant15",
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -379,20 +375,12 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/input-log-secret.yaml"}, tenantId = "tenant16")
|
||||
void shouldNotLogSecretInput() throws TimeoutException, QueueException, InterruptedException {
|
||||
AtomicReference<LogEntry> logEntry = new AtomicReference<>();
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {
|
||||
LogEntry left = l.getLeft();
|
||||
if (left.getTenantId().equals("tenant16")){
|
||||
logEntry.set(left);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
@LoadFlows({"flows/valids/input-log-secret.yaml"})
|
||||
void shouldNotLogSecretInput() throws TimeoutException, QueueException {
|
||||
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {});
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
"tenant16",
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"input-log-secret",
|
||||
null,
|
||||
@@ -402,21 +390,20 @@ public class InputsTest {
|
||||
assertThat(execution.getTaskRunList()).hasSize(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
receive.blockLast();
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
assertThat(logEntry.get()).isNotNull();
|
||||
assertThat(logEntry.get().getMessage()).isEqualTo("These are my secrets: ****** - ******");
|
||||
var logEntry = receive.blockLast();
|
||||
assertThat(logEntry).isNotNull();
|
||||
assertThat(logEntry.getMessage()).isEqualTo("These are my secrets: ****** - ******");
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant17")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void fileInputWithFileDefault() throws IOException, QueueException, TimeoutException {
|
||||
HashMap<String, Object> newInputs = new HashMap<>(InputsTest.inputs);
|
||||
URI file = createFile();
|
||||
newInputs.put("file", file);
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
"tenant17",
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -428,14 +415,14 @@ public class InputsTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant18")
|
||||
@LoadFlows({"flows/valids/inputs.yaml"})
|
||||
void fileInputWithNsfile() throws IOException, QueueException, TimeoutException {
|
||||
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
|
||||
URI file = createNsFile(false, "tenant18");
|
||||
URI file = createNsFile(false);
|
||||
inputs.put("file", file);
|
||||
|
||||
Execution execution = runnerUtils.runOne(
|
||||
"tenant18",
|
||||
MAIN_TENANT,
|
||||
"io.kestra.tests",
|
||||
"inputs",
|
||||
null,
|
||||
@@ -452,11 +439,11 @@ public class InputsTest {
|
||||
return tempFile.toPath().toUri();
|
||||
}
|
||||
|
||||
private URI createNsFile(boolean nsInAuthority, String tenantId) throws IOException {
|
||||
private URI createNsFile(boolean nsInAuthority) throws IOException {
|
||||
String namespace = "io.kestra.tests";
|
||||
String filePath = "file.txt";
|
||||
storageInterface.createDirectory(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storageInterface.put(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
storageInterface.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
|
||||
storageInterface.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
|
||||
return URI.create("nsfile://" + (nsInAuthority ? namespace : "") + "/" + filePath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,17 +14,15 @@ import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
@KestraTest(startRunner = true)
|
||||
class ListenersTest {
|
||||
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
private LocalFlowRepositoryLoader repositoryLoader;
|
||||
|
||||
@@ -1,193 +1,244 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.flows.State.Type;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.repositories.ArrayListTotal;
|
||||
import io.kestra.core.repositories.ExecutionRepositoryInterface;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueFactoryInterface;
|
||||
import io.kestra.core.queues.QueueInterface;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
|
||||
import io.micronaut.data.model.Pageable;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Named;
|
||||
import jakarta.inject.Singleton;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@Singleton
|
||||
public class MultipleConditionTriggerCaseTest {
|
||||
|
||||
public static final String NAMESPACE = "io.kestra.tests.trigger";
|
||||
@Inject
|
||||
@Named(QueueFactoryInterface.EXECUTION_NAMED)
|
||||
protected QueueInterface<Execution> executionQueue;
|
||||
|
||||
@Inject
|
||||
protected TestRunnerUtils runnerUtils;
|
||||
protected RunnerUtils runnerUtils;
|
||||
|
||||
@Inject
|
||||
protected FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
protected ExecutionRepositoryInterface executionRepository;
|
||||
|
||||
@Inject
|
||||
protected ApplicationContext applicationContext;
|
||||
|
||||
public void trigger() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(3);
|
||||
ConcurrentHashMap<String, Execution> ended = new ConcurrentHashMap<>();
|
||||
List<String> watchedExecutions = List.of("trigger-multiplecondition-flow-a",
|
||||
"trigger-multiplecondition-flow-b",
|
||||
"trigger-multiplecondition-listener"
|
||||
);
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (watchedExecutions.contains(execution.getFlowId()) && execution.getState().getCurrent() == State.Type.SUCCESS) {
|
||||
ended.put(execution.getId(), execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// first one
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-a");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-a", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// wait a little to be sure that the trigger is not launching execution
|
||||
Thread.sleep(1000);
|
||||
ArrayListTotal<Execution> flowBExecutions = executionRepository.findByFlowId(MAIN_TENANT,
|
||||
NAMESPACE, "trigger-multiplecondition-flow-b", Pageable.UNPAGED);
|
||||
ArrayListTotal<Execution> listenerExecutions = executionRepository.findByFlowId(MAIN_TENANT,
|
||||
NAMESPACE, "trigger-multiplecondition-listener", Pageable.UNPAGED);
|
||||
assertThat(flowBExecutions).isEmpty();
|
||||
assertThat(listenerExecutions).isEmpty();
|
||||
assertThat(ended.size()).isEqualTo(1);
|
||||
|
||||
// second one
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-b");
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-b", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-listener");
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(ended.size()).isEqualTo(3);
|
||||
|
||||
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-listener").orElseThrow();
|
||||
Execution triggerExecution = ended.entrySet()
|
||||
.stream()
|
||||
.filter(e -> e.getValue().getFlowId().equals(flow.getId()))
|
||||
.findFirst()
|
||||
.map(Map.Entry::getValue)
|
||||
.orElseThrow();
|
||||
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo(
|
||||
NAMESPACE);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-multiplecondition-flow-b");
|
||||
}
|
||||
|
||||
public void failed(String tenantId) throws InterruptedException, TimeoutException, QueueException {
|
||||
public void failed() throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> listener = new AtomicReference<>();
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getFlowId().equals("trigger-flow-listener-namespace-condition")
|
||||
&& execution.getState().getCurrent().isTerminated()) {
|
||||
listener.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// first one
|
||||
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-c");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-c", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
|
||||
|
||||
// wait a little to be sure that the trigger is not launching execution
|
||||
Thread.sleep(1000);
|
||||
ArrayListTotal<Execution> byFlowId = executionRepository.findByFlowId(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-d", Pageable.UNPAGED);
|
||||
assertThat(byFlowId).isEmpty();
|
||||
assertThat(listener.get()).isNull();
|
||||
|
||||
// second one
|
||||
execution = runnerUtils.runOne(tenantId, NAMESPACE,
|
||||
"trigger-multiplecondition-flow-d");
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
|
||||
"trigger-multiplecondition-flow-d", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
tenantId, NAMESPACE, "trigger-flow-listener-namespace-condition");
|
||||
|
||||
// trigger was not done
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(listener.get()).isNotNull();
|
||||
assertThat(listener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void flowTriggerPreconditions() throws TimeoutException, QueueException {
|
||||
public void flowTriggerPreconditions()
|
||||
throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-preconditions-flow-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// flowA
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// flowB: we trigger it two times, as flow-trigger-flow-preconditions-flow-listen is configured with resetOnSuccess: false it should be triggered two times
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b");
|
||||
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
|
||||
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
|
||||
}
|
||||
|
||||
public void flowTriggerPreconditionsMergeOutputs(String tenantId) throws QueueException, TimeoutException {
|
||||
public void flowTriggerPreconditionsMergeOutputs() throws QueueException, TimeoutException, InterruptedException {
|
||||
// we do the same as in flowTriggerPreconditions() but we trigger flows in the opposite order to be sure that outputs are merged
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-preconditions-flow-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
// flowB
|
||||
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b");
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// flowA
|
||||
execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a");
|
||||
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
|
||||
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
tenantId, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
|
||||
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
|
||||
}
|
||||
|
||||
public void flowTriggerOnPaused() throws TimeoutException, QueueException {
|
||||
public void flowTriggerOnPaused()
|
||||
throws InterruptedException, TimeoutException, QueueException {
|
||||
CountDownLatch countDownLatch = new CountDownLatch(1);
|
||||
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
|
||||
|
||||
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
|
||||
Execution execution = either.getLeft();
|
||||
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
|
||||
.equals("flow-trigger-paused-listen")) {
|
||||
flowTrigger.set(execution);
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
});
|
||||
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.paused",
|
||||
"flow-trigger-paused-flow");
|
||||
"flow-trigger-paused-flow", Duration.ofSeconds(60));
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(2);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
Execution triggerExecution = runnerUtils.awaitFlowExecution(
|
||||
e -> e.getState().getCurrent().equals(Type.SUCCESS),
|
||||
MAIN_TENANT, "io.kestra.tests.trigger.paused", "flow-trigger-paused-listen");
|
||||
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
|
||||
receive.blockLast();
|
||||
assertThat(flowTrigger.get()).isNotNull();
|
||||
|
||||
Execution triggerExecution = flowTrigger.get();
|
||||
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
|
||||
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
}
|
||||
|
||||
public void forEachItemWithFlowTrigger() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.foreachitem",
|
||||
"flow-trigger-for-each-item-parent");
|
||||
assertThat(execution.getTaskRunList().size()).isEqualTo(5);
|
||||
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
|
||||
// trigger is done
|
||||
List<Execution> childExecutions = runnerUtils.awaitFlowExecutionNumber(5, MAIN_TENANT, "io.kestra.tests.trigger.foreachitem", "flow-trigger-for-each-item-child");
|
||||
assertThat(childExecutions).hasSize(5);
|
||||
childExecutions.forEach(exec -> {
|
||||
assertThat(exec.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(exec.getTaskRunList().size()).isEqualTo(1);
|
||||
});
|
||||
|
||||
List<Execution> grandchildExecutions = runnerUtils.awaitFlowExecutionNumber(5, MAIN_TENANT, "io.kestra.tests.trigger.foreachitem", "flow-trigger-for-each-item-grandchild");
|
||||
assertThat(grandchildExecutions).hasSize(5);
|
||||
grandchildExecutions.forEach(exec -> {
|
||||
assertThat(exec.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
|
||||
assertThat(exec.getTaskRunList().size()).isEqualTo(2);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@Singleton
|
||||
public class PluginDefaultsCaseTest {
|
||||
@Inject
|
||||
private TestRunnerUtils runnerUtils;
|
||||
private RunnerUtils runnerUtils;
|
||||
|
||||
public void taskDefaults() throws TimeoutException, QueueException {
|
||||
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "plugin-defaults", Duration.ofSeconds(60));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user