mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-25 11:12:12 -05:00
Compare commits
157 Commits
executor_v
...
v1.0.9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e54183a44 | ||
|
|
8aa332c629 | ||
|
|
d10893ca00 | ||
|
|
c5ef356a1c | ||
|
|
0313e8e49b | ||
|
|
f4b6161f14 | ||
|
|
e69e82a35e | ||
|
|
e77378bcb7 | ||
|
|
3c9df90a35 | ||
|
|
6c86f0917c | ||
|
|
30b7346ee0 | ||
|
|
2f485c74ff | ||
|
|
3a5713bbd1 | ||
|
|
2eed738b83 | ||
|
|
5e2609ce5e | ||
|
|
86f909ce93 | ||
|
|
a8cb28a127 | ||
|
|
0fe9ba3e13 | ||
|
|
40f5aadd1a | ||
|
|
ceac25429a | ||
|
|
4144d9fbb1 | ||
|
|
9cc7d45f74 | ||
|
|
81ee330b9e | ||
|
|
5382655a2e | ||
|
|
483f7dc3b2 | ||
|
|
3c2da63837 | ||
|
|
31527891b2 | ||
|
|
6364f419d9 | ||
|
|
3c14432412 | ||
|
|
eaea4f5012 | ||
|
|
d43390a579 | ||
|
|
2404c36d35 | ||
|
|
bdbd217171 | ||
|
|
019c16af3c | ||
|
|
ff7d7c6a0b | ||
|
|
1042be87da | ||
|
|
104805d780 | ||
|
|
33c8e54f36 | ||
|
|
ff2e00d1ca | ||
|
|
0fe3f317c7 | ||
|
|
f753d15c91 | ||
|
|
c03e31de68 | ||
|
|
9a79f9a64c | ||
|
|
41468652d4 | ||
|
|
bc182277de | ||
|
|
8c2271089c | ||
|
|
9973a2120b | ||
|
|
bdfd038d40 | ||
|
|
a3fd734082 | ||
|
|
553a1d5389 | ||
|
|
c58aca967b | ||
|
|
27dcf60770 | ||
|
|
4e7c75232a | ||
|
|
f452da7ce1 | ||
|
|
43401c5017 | ||
|
|
067b110cf0 | ||
|
|
4ceff83a28 | ||
|
|
5026afe5bf | ||
|
|
3c899fcb2f | ||
|
|
cee412ffa9 | ||
|
|
3a57a683be | ||
|
|
a0b9de934e | ||
|
|
d677317cc5 | ||
|
|
9e661195e5 | ||
|
|
09c921bee5 | ||
|
|
d21ec4e899 | ||
|
|
efdb25fa97 | ||
|
|
37bdcc342c | ||
|
|
6d35f2b7a6 | ||
|
|
fe46ddf381 | ||
|
|
359dc9adc0 | ||
|
|
39c930124f | ||
|
|
1686fc3b4e | ||
|
|
03ff25ff55 | ||
|
|
d02fd53287 | ||
|
|
6c16bbe853 | ||
|
|
aa7a473d49 | ||
|
|
95133ebc40 | ||
|
|
54482e1d06 | ||
|
|
54b7811812 | ||
|
|
050ad60a95 | ||
|
|
030627ba7b | ||
|
|
c06ef7958f | ||
|
|
692d046289 | ||
|
|
92c1f04ec0 | ||
|
|
9e11d5fe5e | ||
|
|
14952c9457 | ||
|
|
ae314c301d | ||
|
|
f8aa5fb6ba | ||
|
|
c87d7e4da0 | ||
|
|
c928f1d822 | ||
|
|
baa07dd02b | ||
|
|
260cb50651 | ||
|
|
0a45325c69 | ||
|
|
c2522e2544 | ||
|
|
27476279ae | ||
|
|
3cc6372cb5 | ||
|
|
5f6e9dbe06 | ||
|
|
5078ce741d | ||
|
|
b7e17b7114 | ||
|
|
acaee34b0e | ||
|
|
1d78332505 | ||
|
|
7249632510 | ||
|
|
4a66a08c3b | ||
|
|
22fd6e97ea | ||
|
|
9afd86d32b | ||
|
|
797ea6c9e4 | ||
|
|
07d5e815c4 | ||
|
|
33ac9b1495 | ||
|
|
4d5b95d040 | ||
|
|
667aca7345 | ||
|
|
e05cc65202 | ||
|
|
71b606c27c | ||
|
|
47f9f12ce8 | ||
|
|
01acae5e97 | ||
|
|
e5878f08b7 | ||
|
|
0bcb6b4e0d | ||
|
|
3c2ecf4342 | ||
|
|
3d4f66772e | ||
|
|
e2afd4bcc3 | ||
|
|
d143097f03 | ||
|
|
72c0d91c1a | ||
|
|
1d692e56b0 | ||
|
|
0352d617ac | ||
|
|
b41aa4e0b9 | ||
|
|
d811dc030b | ||
|
|
105e62eee1 | ||
|
|
28796862a4 | ||
|
|
637cd794a4 | ||
|
|
fdd5c6e63d | ||
|
|
eda2483ec9 | ||
|
|
7b3c296489 | ||
|
|
fe6f8b4ed9 | ||
|
|
17ff539690 | ||
|
|
bbd0dda47e | ||
|
|
27a8e8b5a7 | ||
|
|
d6620a34cd | ||
|
|
6f8b3c5cfd | ||
|
|
6da6cbab60 | ||
|
|
a899e16178 | ||
|
|
568cd0b0c7 | ||
|
|
92e1dcb6eb | ||
|
|
499e040cd0 | ||
|
|
5916831d62 | ||
|
|
0b1b55957e | ||
|
|
7ee40d376a | ||
|
|
e2c9b3e256 | ||
|
|
556730777b | ||
|
|
c1a75a431f | ||
|
|
4a5b91667a | ||
|
|
f7b2af16a1 | ||
|
|
9351cb22e0 | ||
|
|
b1ecb82fdc | ||
|
|
c6d56151eb | ||
|
|
ed4398467a | ||
|
|
c51947419a | ||
|
|
ccb6a1f4a7 |
2
.github/ISSUE_TEMPLATE/bug.yml
vendored
2
.github/ISSUE_TEMPLATE/bug.yml
vendored
@@ -4,7 +4,7 @@ body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack). Don't forget to give us a star! ⭐
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack).
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Describe the issue
|
||||
|
||||
2
.github/ISSUE_TEMPLATE/feature.yml
vendored
2
.github/ISSUE_TEMPLATE/feature.yml
vendored
@@ -4,7 +4,7 @@ body:
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Feature description
|
||||
placeholder: Tell us more about your feature request. Don't forget to give us a star! ⭐
|
||||
placeholder: Tell us more about your feature request
|
||||
validations:
|
||||
required: true
|
||||
labels:
|
||||
|
||||
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@@ -35,4 +35,4 @@ Remove this section if this change applies to all flows or to the documentation
|
||||
|
||||
If there are no setup requirements, you can remove this section.
|
||||
|
||||
Thank you for your contribution. ❤️ Don't forget to give us a star! ⭐ -->
|
||||
Thank you for your contribution. ❤️ -->
|
||||
|
||||
67
.github/workflows/auto-translate-ui-keys.yml
vendored
67
.github/workflows/auto-translate-ui-keys.yml
vendored
@@ -1,67 +0,0 @@
|
||||
name: Auto-Translate UI keys and create PR
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9-21/3 * * *" # Every 3 hours from 9 AM to 9 PM
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
retranslate_modified_keys:
|
||||
description: "Whether to re-translate modified keys even if they already have translations."
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
default: "false"
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
translations:
|
||||
name: Translations
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
name: Checkout
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: pip install gitpython openai
|
||||
|
||||
- name: Generate translations
|
||||
run: python ui/src/translations/generate_translations.py ${{ github.event.inputs.retranslate_modified_keys }}
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: "20.x"
|
||||
|
||||
- name: Set up Git
|
||||
run: |
|
||||
git config --global user.name "GitHub Action"
|
||||
git config --global user.email "actions@github.com"
|
||||
|
||||
- name: Commit and create PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH_NAME="chore/update-translations-$(date +%s)"
|
||||
git checkout -b $BRANCH_NAME
|
||||
git add ui/src/translations/*.json
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit. Exiting with success."
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "chore(core): localize to languages other than english" -m "Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference."
|
||||
git push -u origin $BRANCH_NAME || (git push origin --delete $BRANCH_NAME && git push -u origin $BRANCH_NAME)
|
||||
gh pr create --title "Translations from en.json" --body $'This PR was created automatically by a GitHub Action.\n\nSomeone from the @kestra-io/frontend team needs to review and merge.' --base ${{ github.ref_name }} --head $BRANCH_NAME
|
||||
|
||||
- name: Check keys matching
|
||||
run: node ui/src/translations/check.js
|
||||
85
.github/workflows/codeql-analysis.yml
vendored
85
.github/workflows/codeql-analysis.yml
vendored
@@ -1,85 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 5 * * 1'
|
||||
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# Override automatic language detection by changing the below list
|
||||
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
|
||||
language: ['java', 'javascript']
|
||||
# Learn more...
|
||||
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
# We must fetch at least the immediate parents so that if this is
|
||||
# a pull request then we can checkout the head.
|
||||
fetch-depth: 2
|
||||
|
||||
# If this run was triggered by a pull request event, then checkout
|
||||
# the head of the pull request instead of the merge commit.
|
||||
- run: git checkout HEAD^2
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Set up JDK
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: 21
|
||||
|
||||
- name: Setup gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
|
||||
- name: Build with Gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
run: ./gradlew testClasses -x :ui:assembleFrontend
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
if: ${{ matrix.language != 'java' }}
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
15
.github/workflows/e2e-scheduling.yml
vendored
15
.github/workflows/e2e-scheduling.yml
vendored
@@ -1,15 +0,0 @@
|
||||
name: 'E2E tests scheduling'
|
||||
# 'New E2E tests implementation started by Roman. Based on playwright in npm UI project, tests Kestra OSS develop docker image. These tests are written from zero, lets make them unflaky from the start!.'
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 * * * *" # Every hour
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
jobs:
|
||||
e2e:
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-e2e-tests.yml@main
|
||||
@@ -1,84 +0,0 @@
|
||||
name: Create new release branch
|
||||
run-name: "Create new release branch Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
nextVersion:
|
||||
description: 'The next version (e.g., 0.22.0-SNAPSHOT)'
|
||||
required: true
|
||||
type: string
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
NEXT_VERSION: "${{ github.event.inputs.nextVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
steps:
|
||||
# Checks
|
||||
- name: Check Inputs
|
||||
run: |
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)\.0$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$NEXT_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$ ]]; then
|
||||
echo "Invalid next version. Must match regex: ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$"
|
||||
exit 1;
|
||||
fi
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: kestra
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
caches-enabled: true
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
PUSH_RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
|
||||
cd kestra
|
||||
|
||||
# Create and push release branch
|
||||
git checkout -b "$PUSH_RELEASE_BRANCH";
|
||||
git push -u origin "$PUSH_RELEASE_BRANCH";
|
||||
|
||||
# Run gradle release
|
||||
git checkout develop;
|
||||
|
||||
if [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]]; then
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}" \
|
||||
-Prelease.failOnSnapshotDependencies=false
|
||||
else
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}"
|
||||
fi
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Run Gradle Release for Kestra Plugins
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
nextVersion:
|
||||
description: 'The next version (e.g., 0.22.0-SNAPSHOT)'
|
||||
required: true
|
||||
type: string
|
||||
dryRun:
|
||||
description: 'Use DRY_RUN mode'
|
||||
required: false
|
||||
default: 'false'
|
||||
jobs:
|
||||
release:
|
||||
name: Release plugins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
|
||||
- name: 'Configure Git'
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
if: ${{ github.event.inputs.dryRun == 'false' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
|
||||
- name: Run Gradle Release (DRY_RUN)
|
||||
if: ${{ github.event.inputs.dryRun == 'true' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/release-plugins.sh;
|
||||
|
||||
./dev-tools/release-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--next-version=${{github.event.inputs.nextVersion}} \
|
||||
--dry-run \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
@@ -1,60 +0,0 @@
|
||||
name: Set Version and Tag Plugins
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
dryRun:
|
||||
description: 'Use DRY_RUN mode'
|
||||
required: false
|
||||
default: 'false'
|
||||
jobs:
|
||||
tag:
|
||||
name: Release plugins
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Get Plugins List
|
||||
- name: Get Plugins List
|
||||
uses: kestra-io/actions/composite/kestra-oss/kestra-oss-plugins-list@main
|
||||
id: plugins-list
|
||||
with:
|
||||
plugin-version: 'LATEST'
|
||||
|
||||
- name: 'Configure Git'
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Set Version and Tag Plugins
|
||||
if: ${{ github.event.inputs.dryRun == 'false' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
|
||||
- name: Set Version and Tag Plugins (DRY_RUN)
|
||||
if: ${{ github.event.inputs.dryRun == 'true' }}
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
chmod +x ./dev-tools/setversion-tag-plugins.sh;
|
||||
|
||||
./dev-tools/setversion-tag-plugins.sh \
|
||||
--release-version=${{github.event.inputs.releaseVersion}} \
|
||||
--dry-run \
|
||||
--yes \
|
||||
${{ steps.plugins-list.outputs.repositories }}
|
||||
65
.github/workflows/global-start-release.yml
vendored
65
.github/workflows/global-start-release.yml
vendored
@@ -1,65 +0,0 @@
|
||||
name: Start release
|
||||
run-name: "Start release of Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.1)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Parse and Check Inputs
|
||||
id: parse-and-check-inputs
|
||||
run: |
|
||||
CURRENT_BRANCH="${{ github.ref_name }}"
|
||||
if ! [[ "$CURRENT_BRANCH" == "develop" ]]; then
|
||||
echo "You can only run this workflow on develop, but you ran it on $CURRENT_BRANCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)(\.[0-9]+)(-rc[0-9])?(-SNAPSHOT)?$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
echo "release_branch=${RELEASE_BRANCH}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Checkout
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
ref: ${{ steps.parse-and-check-inputs.outputs.release_branch }}
|
||||
|
||||
# Configure
|
||||
- name: Git - Configure
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Start release by updating version and pushing a new tag
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Update version
|
||||
sed -i "s/^version=.*/version=$RELEASE_VERSION/" ./gradle.properties
|
||||
git add ./gradle.properties
|
||||
git commit -m"chore(version): update to version '$RELEASE_VERSION'"
|
||||
git push
|
||||
git tag -a "v$RELEASE_VERSION" -m"v$RELEASE_VERSION"
|
||||
git push --tags
|
||||
9
.github/workflows/main-build.yml
vendored
9
.github/workflows/main-build.yml
vendored
@@ -67,13 +67,12 @@ jobs:
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [backend-tests, frontend-tests, publish-develop-docker, publish-develop-maven]
|
||||
needs: [publish-develop-docker, publish-develop-maven]
|
||||
if: always()
|
||||
steps:
|
||||
- run: echo "debug repo ${{github.repository}} ref ${{github.ref}} res ${{needs.publish-develop-maven.result}} jobStatus ${{job.status}} isNotFork ${{github.repository == 'kestra-io/kestra'}} isDevelop ${{github.ref == 'refs/heads/develop'}}"
|
||||
- name: Trigger EE Workflow
|
||||
uses: peter-evans/repository-dispatch@v4
|
||||
if: github.ref == 'refs/heads/develop' && needs.publish-develop-maven == 'success'
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
if: github.ref == 'refs/heads/develop' && needs.release.result == 'success'
|
||||
with:
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
repository: kestra-io/kestra-ee
|
||||
@@ -81,7 +80,7 @@ jobs:
|
||||
|
||||
# Slack
|
||||
- name: Slack - Notification
|
||||
if: ${{ failure() && github.repository == 'kestra-io/kestra' && (github.ref == 'refs/heads/develop') }}
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
11
.github/workflows/pre-release.yml
vendored
11
.github/workflows/pre-release.yml
vendored
@@ -5,6 +5,15 @@ on:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
skip-test:
|
||||
description: 'Skip test'
|
||||
type: choice
|
||||
required: true
|
||||
default: 'false'
|
||||
options:
|
||||
- "true"
|
||||
- "false"
|
||||
|
||||
jobs:
|
||||
build-artifacts:
|
||||
@@ -14,6 +23,7 @@ jobs:
|
||||
backend-tests:
|
||||
name: Backend tests
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-backend-tests.yml@main
|
||||
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
@@ -23,6 +33,7 @@ jobs:
|
||||
frontend-tests:
|
||||
name: Frontend tests
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-frontend-tests.yml@main
|
||||
if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
|
||||
secrets:
|
||||
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
12
.github/workflows/release-docker.yml
vendored
12
.github/workflows/release-docker.yml
vendored
@@ -13,11 +13,11 @@ on:
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
plugin-version:
|
||||
description: 'Plugin version'
|
||||
required: false
|
||||
type: string
|
||||
default: "LATEST"
|
||||
dry-run:
|
||||
description: 'Dry run mode that will not write or release anything'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
publish-docker:
|
||||
@@ -25,9 +25,9 @@ jobs:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-publish-docker.yml@main
|
||||
with:
|
||||
plugin-version: ${{ inputs.plugin-version }}
|
||||
retag-latest: ${{ inputs.retag-latest }}
|
||||
retag-lts: ${{ inputs.retag-lts }}
|
||||
dry-run: ${{ inputs.dry-run }}
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
|
||||
74
.github/workflows/vulnerabilities-check.yml
vendored
74
.github/workflows/vulnerabilities-check.yml
vendored
@@ -48,77 +48,3 @@ jobs:
|
||||
with:
|
||||
name: dependency-check-report
|
||||
path: build/reports/dependency-check-report.html
|
||||
|
||||
develop-image-check:
|
||||
name: Image Check (develop)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:develop
|
||||
format: 'template'
|
||||
template: '@/contrib/sarif.tpl'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
skip-dirs: /app/plugins
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
|
||||
latest-image-check:
|
||||
name: Image Check (latest)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:latest
|
||||
format: table
|
||||
skip-dirs: /app/plugins
|
||||
scanners: vuln
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
13
README.md
13
README.md
@@ -19,12 +19,9 @@
|
||||
<br />
|
||||
|
||||
<p align="center">
|
||||
<a href="https://twitter.com/kestra_io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/twitter.svg" alt="twitter" width="35" height="25" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" width="35" height="25" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/youtube.svg" alt="youtube" width="35" height="25" /></a>
|
||||
<a href="https://x.com/kestra_io"><img height="25" src="https://kestra.io/twitter.svg" alt="X(formerly Twitter)" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/"><img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io"><img height="25" src="https://kestra.io/youtube.svg" alt="youtube" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -36,10 +33,10 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.kestra.io/video/product-overview" target="_blank">
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
|
||||
</a>
|
||||
</p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
|
||||
|
||||
|
||||
## 🌟 What is Kestra?
|
||||
|
||||
21
build.gradle
21
build.gradle
@@ -25,19 +25,19 @@ plugins {
|
||||
id 'jacoco-report-aggregation'
|
||||
|
||||
// helper
|
||||
id "com.github.ben-manes.versions" version "0.53.0"
|
||||
id "com.github.ben-manes.versions" version "0.52.0"
|
||||
|
||||
// front
|
||||
id 'com.github.node-gradle.node' version '7.1.0'
|
||||
|
||||
// release
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.3"
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.2"
|
||||
id 'signing'
|
||||
id "com.vanniktech.maven.publish" version "0.34.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.6" apply false
|
||||
id "org.owasp.dependencycheck" version "12.1.3" apply false
|
||||
}
|
||||
|
||||
idea {
|
||||
@@ -168,9 +168,8 @@ allprojects {
|
||||
/**********************************************************************************************************************\
|
||||
* Test
|
||||
**********************************************************************************************************************/
|
||||
subprojects {subProj ->
|
||||
|
||||
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
|
||||
subprojects {
|
||||
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
|
||||
apply plugin: "com.adarshr.test-logger"
|
||||
|
||||
java {
|
||||
@@ -222,14 +221,6 @@ subprojects {subProj ->
|
||||
t.environment 'ENV_TEST1', "true"
|
||||
t.environment 'ENV_TEST2', "Pass by env"
|
||||
|
||||
|
||||
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
|
||||
// JUnit 5 parallel settings
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('flakyTest', Test) { Test t ->
|
||||
@@ -372,7 +363,7 @@ tasks.named('testCodeCoverageReport') {
|
||||
subprojects {
|
||||
sonar {
|
||||
properties {
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml,$projectDir.parentFile.path/build/reports/jacoco/test//testCodeCoverageReport.xml"
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,5 @@ dependencies {
|
||||
implementation project(":worker")
|
||||
|
||||
//test
|
||||
testImplementation project(':tests')
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
@@ -117,7 +117,7 @@ public abstract class AbstractValidateCommand extends AbstractApiCommand {
|
||||
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows/validate", tenantService.getTenantId(tenantId)), body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows/validate", tenantService.getTenantIdAndAllowEETenants(tenantId)), body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
List<ValidateConstraintViolation> validations = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -43,13 +43,13 @@ import java.util.concurrent.Callable;
|
||||
SysCommand.class,
|
||||
ConfigCommand.class,
|
||||
NamespaceCommand.class,
|
||||
MigrationCommand.class
|
||||
MigrationCommand.class,
|
||||
}
|
||||
)
|
||||
@Introspected
|
||||
public class App implements Callable<Integer> {
|
||||
public static void main(String[] args) {
|
||||
execute(App.class, new String [] { Environment.CLI }, args);
|
||||
execute(App.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
|
||||
return PicocliRunner.call(App.class, "--help");
|
||||
}
|
||||
|
||||
protected static void execute(Class<?> cls, String[] environments, String... args) {
|
||||
protected static void execute(Class<?> cls, String... args) {
|
||||
// Log Bridge
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
|
||||
// Init ApplicationContext
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, args);
|
||||
|
||||
// Call Picocli command
|
||||
int exitCode = 0;
|
||||
@@ -80,7 +80,6 @@ public class App implements Callable<Integer> {
|
||||
System.exit(Objects.requireNonNullElse(exitCode, 0));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
|
||||
* forced Properties from current command.
|
||||
@@ -89,13 +88,12 @@ public class App implements Callable<Integer> {
|
||||
* @return the application context created
|
||||
*/
|
||||
protected static ApplicationContext applicationContext(Class<?> mainClass,
|
||||
String[] environments,
|
||||
String[] args) {
|
||||
|
||||
ApplicationContextBuilder builder = ApplicationContext
|
||||
.builder()
|
||||
.mainClass(mainClass)
|
||||
.environments(environments);
|
||||
.environments(Environment.CLI);
|
||||
|
||||
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
|
||||
continueOnParsingErrors(cmd);
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.runners.*;
|
||||
import io.kestra.core.server.Service;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.ExecutorsUtils;
|
||||
import io.kestra.executor.DefaultExecutor;
|
||||
import io.kestra.worker.DefaultWorker;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.context.annotation.Value;
|
||||
@@ -50,7 +49,7 @@ public class StandAloneRunner implements Runnable, AutoCloseable {
|
||||
running.set(true);
|
||||
|
||||
poolExecutor = executorsUtils.cachedThreadPool("standalone-runner");
|
||||
poolExecutor.execute(applicationContext.getBean(DefaultExecutor.class));
|
||||
poolExecutor.execute(applicationContext.getBean(ExecutorInterface.class));
|
||||
|
||||
if (workerEnabled) {
|
||||
// FIXME: For backward-compatibility with Kestra 0.15.x and earliest we still used UUID for Worker ID instead of IdUtils
|
||||
|
||||
@@ -24,7 +24,8 @@ public class FlowValidateCommand extends AbstractValidateCommand {
|
||||
private FlowService flowService;
|
||||
|
||||
@Inject
|
||||
private TenantIdSelectorService tenantService;
|
||||
private TenantIdSelectorService tenantIdSelectorService;
|
||||
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
@@ -39,7 +40,7 @@ public class FlowValidateCommand extends AbstractValidateCommand {
|
||||
FlowWithSource flow = (FlowWithSource) object;
|
||||
List<String> warnings = new ArrayList<>();
|
||||
warnings.addAll(flowService.deprecationPaths(flow).stream().map(deprecation -> deprecation + " is deprecated").toList());
|
||||
warnings.addAll(flowService.warnings(flow, tenantService.getTenantId(tenantId)));
|
||||
warnings.addAll(flowService.warnings(flow, tenantIdSelectorService.getTenantIdAndAllowEETenants(tenantId)));
|
||||
return warnings;
|
||||
},
|
||||
(Object object) -> {
|
||||
|
||||
@@ -64,7 +64,7 @@ public class FlowNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCo
|
||||
}
|
||||
try(DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<String> request = HttpRequest
|
||||
.POST(apiUri("/flows/", tenantService.getTenantId(tenantId)) + namespace + "?delete=" + delete, body).contentType(MediaType.APPLICATION_YAML);
|
||||
.POST(apiUri("/flows/", tenantService.getTenantIdAndAllowEETenants(tenantId)) + namespace + "?delete=" + delete, body).contentType(MediaType.APPLICATION_YAML);
|
||||
|
||||
List<UpdateResult> updated = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -49,7 +49,7 @@ public class NamespaceFilesUpdateCommand extends AbstractApiCommand {
|
||||
|
||||
try (var files = Files.walk(from); DefaultHttpClient client = client()) {
|
||||
if (delete) {
|
||||
client.toBlocking().exchange(this.requestOptions(HttpRequest.DELETE(apiUri("/namespaces/", tenantService.getTenantId(tenantId)) + namespace + "/files?path=" + to, null)));
|
||||
client.toBlocking().exchange(this.requestOptions(HttpRequest.DELETE(apiUri("/namespaces/", tenantService.getTenantIdAndAllowEETenants(tenantId)) + namespace + "/files?path=" + to, null)));
|
||||
}
|
||||
|
||||
KestraIgnore kestraIgnore = new KestraIgnore(from);
|
||||
@@ -67,7 +67,7 @@ public class NamespaceFilesUpdateCommand extends AbstractApiCommand {
|
||||
client.toBlocking().exchange(
|
||||
this.requestOptions(
|
||||
HttpRequest.POST(
|
||||
apiUri("/namespaces/", tenantService.getTenantId(tenantId)) + namespace + "/files?path=" + destination,
|
||||
apiUri("/namespaces/", tenantService.getTenantIdAndAllowEETenants(tenantId)) + namespace + "/files?path=" + destination,
|
||||
body
|
||||
).contentType(MediaType.MULTIPART_FORM_DATA)
|
||||
)
|
||||
|
||||
@@ -2,27 +2,19 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
|
||||
Integer serverPort;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
|
||||
|
||||
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
|
||||
|
||||
return super.call();
|
||||
}
|
||||
|
||||
private long maxMemoryInMB() {
|
||||
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
|
||||
}
|
||||
|
||||
protected static int defaultWorkerThread() {
|
||||
return Runtime.getRuntime().availableProcessors() * 8;
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpda
|
||||
|
||||
try (DefaultHttpClient client = client()) {
|
||||
MutableHttpRequest<List<Template>> request = HttpRequest
|
||||
.POST(apiUri("/templates/", tenantService.getTenantId(tenantId)) + namespace + "?delete=" + delete, templates);
|
||||
.POST(apiUri("/templates/", tenantService.getTenantIdAndAllowEETenants(tenantId)) + namespace + "?delete=" + delete, templates);
|
||||
|
||||
List<UpdateResult> updated = client.toBlocking().retrieve(
|
||||
this.requestOptions(request),
|
||||
|
||||
@@ -262,8 +262,6 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private String getTenantIdFromPath(Path path) {
|
||||
// FIXME there is probably a bug here when a tenant has '_' in its name,
|
||||
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
|
||||
return path.getFileName().toString().split("_")[0];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,4 +16,11 @@ public class TenantIdSelectorService {
|
||||
}
|
||||
return MAIN_TENANT;
|
||||
}
|
||||
|
||||
public String getTenantIdAndAllowEETenants(String tenantId) {
|
||||
if (StringUtils.isNotBlank(tenantId)){
|
||||
return tenantId;
|
||||
}
|
||||
return MAIN_TENANT;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,8 +49,6 @@ micronaut:
|
||||
- /ui/.+
|
||||
- /health
|
||||
- /health/.+
|
||||
- /metrics
|
||||
- /metrics/.+
|
||||
- /prometheus
|
||||
http-version: HTTP_1_1
|
||||
caches:
|
||||
|
||||
@@ -37,7 +37,7 @@ class AppTest {
|
||||
|
||||
final String[] args = new String[]{"server", serverType, "--help"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
|
||||
|
||||
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
|
||||
@@ -52,7 +52,7 @@ class AppTest {
|
||||
|
||||
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
|
||||
|
||||
assertThat(out.toString()).startsWith("Missing required parameters: ");
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package io.kestra.cli.commands.configs.sys;
|
||||
import io.kestra.cli.commands.flows.FlowCreateCommand;
|
||||
import io.kestra.cli.commands.namespaces.kv.KvCommand;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.runtime.server.EmbeddedServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
/**
|
||||
* Verifies CLI behavior without repository configuration:
|
||||
* - Repo-independent commands succeed (e.g. KV with no params).
|
||||
* - Repo-dependent commands fail with a clear error.
|
||||
*/
|
||||
class NoConfigCommandTest {
|
||||
|
||||
@Test
|
||||
void shouldSucceedWithNamespaceKVCommandWithoutParamsAndConfig() {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
System.setOut(new PrintStream(out));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
|
||||
String[] args = {};
|
||||
Integer call = PicocliRunner.call(KvCommand.class, ctx, args);
|
||||
|
||||
assertThat(call).isZero();
|
||||
assertThat(out.toString()).contains("Usage: kestra namespace kv");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailWithCreateFlowCommandWithoutConfig() throws URISyntaxException {
|
||||
URL flowUrl = NoConfigCommandTest.class.getClassLoader().getResource("crudFlow/date.yml");
|
||||
Objects.requireNonNull(flowUrl, "Test flow resource not found");
|
||||
|
||||
Path flowPath = Paths.get(flowUrl.toURI());
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
ByteArrayOutputStream err=new ByteArrayOutputStream();
|
||||
|
||||
System.setOut(new PrintStream(out));
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder()
|
||||
.deduceEnvironment(false)
|
||||
.start()) {
|
||||
|
||||
EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
|
||||
embeddedServer.start();
|
||||
|
||||
String[] createArgs = {
|
||||
"--server",
|
||||
embeddedServer.getURL().toString(),
|
||||
"--user",
|
||||
"myuser:pass:word",
|
||||
flowPath.toString(),
|
||||
};
|
||||
|
||||
Integer exitCode = PicocliRunner.call(FlowCreateCommand.class, ctx, createArgs);
|
||||
|
||||
|
||||
assertThat(exitCode).isNotZero();
|
||||
assertThat(out.toString()).isEmpty();
|
||||
assertThat(err.toString()).contains("No bean of type [io.kestra.core.repositories.FlowRepositoryInterface] exists");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -27,6 +27,26 @@ class FlowValidateCommandTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
// github action kestra-io/validate-action requires being able to validate Flows from OSS CLI against a remote EE instance
|
||||
void runForEEInstance() {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
System.setOut(new PrintStream(out));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
|
||||
String[] args = {
|
||||
"--tenant",
|
||||
"some-ee-tenant",
|
||||
"--local",
|
||||
"src/test/resources/helper/include.yaml"
|
||||
};
|
||||
Integer call = PicocliRunner.call(FlowValidateCommand.class, ctx, args);
|
||||
|
||||
assertThat(call).isZero();
|
||||
assertThat(out.toString()).contains("✓ - io.kestra.cli / include");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void warning() {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
package io.kestra.cli.services;
|
||||
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -19,8 +18,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -58,12 +57,10 @@ class FileChangedEventListenerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void test() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a basic flow
|
||||
String flow = """
|
||||
@@ -76,14 +73,14 @@ class FileChangedEventListenerTest {
|
||||
message: Hello World! 🚀
|
||||
""";
|
||||
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
assertThat(myflow.getTasks()).hasSize(1);
|
||||
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
|
||||
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -91,18 +88,16 @@ class FileChangedEventListenerTest {
|
||||
// delete the flow
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void testWithPluginDefault() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a flow with plugin default
|
||||
String pluginDefault = """
|
||||
@@ -118,14 +113,14 @@ class FileChangedEventListenerTest {
|
||||
values:
|
||||
message: Hello World!
|
||||
""";
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -133,7 +128,7 @@ class FileChangedEventListenerTest {
|
||||
// delete both files
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
|
||||
@@ -84,7 +84,7 @@ dependencies {
|
||||
|
||||
testImplementation "org.testcontainers:testcontainers:1.21.3"
|
||||
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
|
||||
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
/**
|
||||
* Top-level marker interface for Kestra's plugin of type App.
|
||||
*/
|
||||
@@ -18,6 +20,6 @@ public interface AppBlockInterface extends io.kestra.core.models.Plugin {
|
||||
)
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
String getType();
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
/**
|
||||
* Top-level marker interface for Kestra's plugin of type App.
|
||||
*/
|
||||
@@ -18,6 +20,6 @@ public interface AppPluginInterface extends io.kestra.core.models.Plugin {
|
||||
)
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
String getType();
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import com.github.victools.jsonschema.generator.impl.DefinitionKey;
|
||||
import com.github.victools.jsonschema.generator.naming.DefaultSchemaDefinitionNamingStrategy;
|
||||
import com.github.victools.jsonschema.module.jackson.JacksonModule;
|
||||
import com.github.victools.jsonschema.module.jackson.JacksonOption;
|
||||
import com.github.victools.jsonschema.module.jackson.JsonUnwrappedDefinitionProvider;
|
||||
import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule;
|
||||
import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption;
|
||||
import com.github.victools.jsonschema.module.swagger2.Swagger2Module;
|
||||
@@ -45,6 +46,9 @@ import io.swagger.v3.oas.annotations.media.Content;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.lang.reflect.*;
|
||||
import java.time.*;
|
||||
@@ -58,7 +62,9 @@ import static io.kestra.core.docs.AbstractClassDocumentation.required;
|
||||
import static io.kestra.core.serializers.JacksonMapper.MAP_TYPE_REFERENCE;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class JsonSchemaGenerator {
|
||||
|
||||
private static final List<Class<?>> TYPES_RESOLVED_AS_STRING = List.of(Duration.class, LocalTime.class, LocalDate.class, LocalDateTime.class, ZonedDateTime.class, OffsetDateTime.class, OffsetTime.class);
|
||||
private static final List<Class<?>> SUBTYPE_RESOLUTION_EXCLUSION_FOR_PLUGIN_SCHEMA = List.of(Task.class, AbstractTrigger.class);
|
||||
|
||||
@@ -270,8 +276,22 @@ public class JsonSchemaGenerator {
|
||||
.with(Option.DEFINITIONS_FOR_ALL_OBJECTS)
|
||||
.with(Option.DEFINITION_FOR_MAIN_SCHEMA)
|
||||
.with(Option.PLAIN_DEFINITION_KEYS)
|
||||
.with(Option.ALLOF_CLEANUP_AT_THE_END);;
|
||||
|
||||
.with(Option.ALLOF_CLEANUP_AT_THE_END);
|
||||
|
||||
// HACK: Registered a custom JsonUnwrappedDefinitionProvider prior to the JacksonModule
|
||||
// to be able to return an CustomDefinition with an empty node when the ResolvedType can't be found.
|
||||
builder.forTypesInGeneral().withCustomDefinitionProvider(new JsonUnwrappedDefinitionProvider(){
|
||||
@Override
|
||||
public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, SchemaGenerationContext context) {
|
||||
try {
|
||||
return super.provideCustomSchemaDefinition(javaType, context);
|
||||
} catch (NoClassDefFoundError e) {
|
||||
// This error happens when a non-supported plugin type exists in the classpath.
|
||||
log.debug("Cannot create schema definition for type '{}'. Cause: NoClassDefFoundError", javaType.getTypeName());
|
||||
return new CustomDefinition(context.getGeneratorConfig().createObjectNode(), true);
|
||||
}
|
||||
}
|
||||
});
|
||||
if (!draft7) {
|
||||
builder.with(new JacksonModule(JacksonOption.IGNORE_TYPE_INFO_TRANSFORM));
|
||||
} else {
|
||||
@@ -300,6 +320,7 @@ public class JsonSchemaGenerator {
|
||||
// inline some type
|
||||
builder.forTypesInGeneral()
|
||||
.withCustomDefinitionProvider(new CustomDefinitionProviderV2() {
|
||||
|
||||
@Override
|
||||
public CustomDefinition provideCustomSchemaDefinition(ResolvedType javaType, SchemaGenerationContext context) {
|
||||
if (javaType.isInstanceOf(Map.class) || javaType.isInstanceOf(Enum.class)) {
|
||||
|
||||
@@ -6,7 +6,6 @@ import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Schema(description = "A key/value pair that can be attached to a Flow or Execution. Labels are often used to organize and categorize objects.")
|
||||
@@ -44,7 +43,7 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
@@ -59,7 +58,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
@@ -74,7 +72,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
if (map == null || map.isEmpty()) return List.of();
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
@@ -93,14 +90,4 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides predicate for not empty entries.
|
||||
*
|
||||
* @return The non-empty filter
|
||||
*/
|
||||
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
|
||||
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
|
||||
entry.getValue() != null && !entry.getValue().isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,16 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface that can be implemented by classes supporting plugin versioning.
|
||||
*
|
||||
* @see Plugin
|
||||
*/
|
||||
public interface PluginVersioning {
|
||||
|
||||
String TITLE = "Plugin Version";
|
||||
String DESCRIPTION = """
|
||||
Defines the version of the plugin to use.
|
||||
|
||||
The version must follow the Semantic Versioning (SemVer) specification:
|
||||
- A single-digit MAJOR version (e.g., `1`).
|
||||
- A MAJOR.MINOR version (e.g., `1.1`).
|
||||
- A MAJOR.MINOR.PATCH version, optionally with any qualifier
|
||||
(e.g., `1.1.2`, `1.1.0-SNAPSHOT`).
|
||||
""";
|
||||
|
||||
@Schema(
|
||||
title = TITLE,
|
||||
description = DESCRIPTION
|
||||
)
|
||||
@Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)")
|
||||
@Schema(title = "The version of the plugin to use.")
|
||||
String getVersion();
|
||||
}
|
||||
|
||||
@@ -91,12 +91,6 @@ public record QueryFilter(
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX, Op.IN, Op.NOT_IN, Op.PREFIX);
|
||||
}
|
||||
},
|
||||
KIND("kind") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.EQUALS,Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
LABELS("labels") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
@@ -217,7 +211,7 @@ public record QueryFilter(
|
||||
return List.of(
|
||||
Field.QUERY, Field.SCOPE, Field.FLOW_ID, Field.START_DATE, Field.END_DATE,
|
||||
Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER,
|
||||
Field.NAMESPACE,Field.KIND
|
||||
Field.NAMESPACE
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -260,6 +254,18 @@ public record QueryFilter(
|
||||
*
|
||||
* @return List of {@code ResourceField} with resource names, fields, and operations.
|
||||
*/
|
||||
public static List<ResourceField> asResourceList() {
|
||||
return Arrays.stream(values())
|
||||
.map(Resource::toResourceField)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ResourceField toResourceField(Resource resource) {
|
||||
List<FieldOp> fieldOps = resource.supportedField().stream()
|
||||
.map(Resource::toFieldInfo)
|
||||
.toList();
|
||||
return new ResourceField(resource.name().toLowerCase(), fieldOps);
|
||||
}
|
||||
|
||||
private static FieldOp toFieldInfo(Field field) {
|
||||
List<Operation> operations = field.supportedOp().stream()
|
||||
@@ -273,6 +279,9 @@ public record QueryFilter(
|
||||
}
|
||||
}
|
||||
|
||||
public record ResourceField(String name, List<FieldOp> fields) {
|
||||
}
|
||||
|
||||
public record FieldOp(String name, String value, List<Operation> operations) {
|
||||
}
|
||||
|
||||
|
||||
@@ -17,12 +17,31 @@ import java.util.List;
|
||||
@Introspected
|
||||
public class ExecutionUsage {
|
||||
private final List<DailyExecutionStatistics> dailyExecutionsCount;
|
||||
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
|
||||
|
||||
public static ExecutionUsage of(final String tenantId,
|
||||
final ExecutionRepositoryInterface executionRepository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
|
||||
try {
|
||||
dailyTaskRunsCount = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenantId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
true);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
|
||||
}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(executionRepository.dailyStatistics(
|
||||
null,
|
||||
@@ -33,13 +52,28 @@ public class ExecutionUsage {
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null))
|
||||
null,
|
||||
false))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
try {
|
||||
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
true
|
||||
);
|
||||
} catch (UnsupportedOperationException ignored) {}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
@@ -47,8 +81,10 @@ public class ExecutionUsage {
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY
|
||||
DateUtils.GroupType.DAY,
|
||||
false
|
||||
))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ import lombok.experimental.SuperBuilder;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@io.kestra.core.models.annotations.Plugin
|
||||
@SuperBuilder
|
||||
@Getter
|
||||
@@ -20,6 +22,6 @@ import jakarta.validation.constraints.Pattern;
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
public abstract class Condition implements Plugin, Rethrow.PredicateChecked<ConditionContext, InternalException> {
|
||||
@NotNull
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
protected String type;
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@@ -28,7 +30,7 @@ import java.util.Set;
|
||||
public abstract class DataFilter<F extends Enum<F>, C extends ColumnDescriptor<F>> implements io.kestra.core.models.Plugin, IData<F> {
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
private String type;
|
||||
|
||||
private Map<String, C> columns;
|
||||
|
||||
@@ -19,6 +19,8 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@@ -27,7 +29,7 @@ import java.util.Set;
|
||||
public abstract class DataFilterKPI<F extends Enum<F>, C extends ColumnDescriptor<F>> implements io.kestra.core.models.Plugin, IData<F> {
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
private String type;
|
||||
|
||||
private C columns;
|
||||
|
||||
@@ -12,6 +12,8 @@ import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@@ -26,7 +28,7 @@ public abstract class Chart<P extends ChartOption> implements io.kestra.core.mod
|
||||
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
protected String type;
|
||||
|
||||
@Valid
|
||||
|
||||
@@ -496,7 +496,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
if (resolvedFinally != null && (
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailedNoRetry(resolvedTasks, parentTaskRun
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailed(resolvedTasks, parentTaskRun
|
||||
))) {
|
||||
return resolvedFinally;
|
||||
}
|
||||
@@ -584,13 +584,6 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastSubmitted(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
.filter(t -> t.getState().getCurrent() == State.Type.SUBMITTED)
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastRunning(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
@@ -872,18 +865,20 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
* @param e the exception raise
|
||||
* @return new taskRun with updated attempt with logs
|
||||
*/
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
|
||||
TaskRun failed = taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
|
||||
TaskRunAttempt lastAttempt, Exception e) {
|
||||
return new FailedTaskRunWithLog(
|
||||
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
|
||||
taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt
|
||||
.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
)
|
||||
.withState(State.Type.FAILED),
|
||||
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
|
||||
);
|
||||
}
|
||||
@@ -941,7 +936,15 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
for (TaskRun current : taskRuns) {
|
||||
if (!MapUtils.isEmpty(current.getOutputs())) {
|
||||
if (current.getIteration() != null) {
|
||||
taskOutputs = MapUtils.merge(taskOutputs, outputs(current, byIds));
|
||||
Map<String, Object> merged = MapUtils.merge(taskOutputs, outputs(current, byIds));
|
||||
// If one of two of the map is null in the merge() method, we just return the other
|
||||
// And if the not null map is a Variables (= read only), we cast it back to a simple
|
||||
// hashmap to avoid taskOutputs becoming read-only
|
||||
// i.e this happen in nested loopUntil tasks
|
||||
if (merged instanceof Variables) {
|
||||
merged = new HashMap<>(merged);
|
||||
}
|
||||
taskOutputs = merged;
|
||||
} else {
|
||||
taskOutputs.putAll(outputs(current, byIds));
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import io.swagger.v3.oas.annotations.Hidden;
|
||||
import jakarta.annotation.Nullable;
|
||||
import lombok.Builder;
|
||||
import lombok.Value;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.event.Level;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
@@ -120,6 +121,16 @@ public class LogEntry implements DeletedInterface, TenantInterface {
|
||||
return logEntry.getTimestamp().toString() + " " + logEntry.getLevel() + " " + logEntry.getMessage();
|
||||
}
|
||||
|
||||
public static String toPrettyString(LogEntry logEntry, Integer maxMessageSize) {
|
||||
String message;
|
||||
if (maxMessageSize != null && maxMessageSize > 0) {
|
||||
message = StringUtils.truncate(logEntry.getMessage(), maxMessageSize);
|
||||
} else {
|
||||
message = logEntry.getMessage();
|
||||
}
|
||||
return logEntry.getTimestamp().toString() + " " + logEntry.getLevel() + " " + message;
|
||||
}
|
||||
|
||||
public Map<String, String> toMap() {
|
||||
return Stream
|
||||
.of(
|
||||
|
||||
@@ -52,6 +52,7 @@ public class TaskRun implements TenantInterface {
|
||||
|
||||
@With
|
||||
@JsonInclude(JsonInclude.Include.ALWAYS)
|
||||
@Nullable
|
||||
Variables outputs;
|
||||
|
||||
@NotNull
|
||||
@@ -64,7 +65,6 @@ public class TaskRun implements TenantInterface {
|
||||
Boolean dynamic;
|
||||
|
||||
// Set it to true to force execution even if the execution is killed
|
||||
@Nullable
|
||||
@With
|
||||
Boolean forceExecution;
|
||||
|
||||
@@ -217,7 +217,7 @@ public class TaskRun implements TenantInterface {
|
||||
public boolean isSame(TaskRun taskRun) {
|
||||
return this.getId().equals(taskRun.getId()) &&
|
||||
((this.getValue() == null && taskRun.getValue() == null) || (this.getValue() != null && this.getValue().equals(taskRun.getValue()))) &&
|
||||
((this.getIteration() == null && taskRun.getIteration() == null) || (this.getIteration() != null && this.getIteration().equals(taskRun.getIteration()))) ;
|
||||
((this.getIteration() == null && taskRun.getIteration() == null) || (this.getIteration() != null && this.getIteration().equals(taskRun.getIteration())));
|
||||
}
|
||||
|
||||
public String toString(boolean pretty) {
|
||||
@@ -249,7 +249,7 @@ public class TaskRun implements TenantInterface {
|
||||
* This method is used when the retry is apply on a task
|
||||
* but the retry type is NEW_EXECUTION
|
||||
*
|
||||
* @param retry Contains the retry configuration
|
||||
* @param retry Contains the retry configuration
|
||||
* @param execution Contains the attempt number and original creation date
|
||||
* @return The next retry date, null if maxAttempt || maxDuration is reached
|
||||
*/
|
||||
@@ -270,6 +270,7 @@ public class TaskRun implements TenantInterface {
|
||||
|
||||
/**
|
||||
* This method is used when the Retry definition comes from the flow
|
||||
*
|
||||
* @param retry The retry configuration
|
||||
* @return The next retry date, null if maxAttempt || maxDuration is reached
|
||||
*/
|
||||
@@ -296,7 +297,7 @@ public class TaskRun implements TenantInterface {
|
||||
}
|
||||
|
||||
public TaskRun incrementIteration() {
|
||||
int iteration = this.iteration == null ? 0 : this.iteration;
|
||||
int iteration = this.iteration == null ? 1 : this.iteration;
|
||||
return this.toBuilder()
|
||||
.iteration(iteration + 1)
|
||||
.build();
|
||||
|
||||
@@ -77,14 +77,6 @@ public abstract class AbstractFlow implements FlowInterface {
|
||||
Map<String, Object> variables;
|
||||
|
||||
|
||||
@Schema(
|
||||
oneOf = {
|
||||
String.class, // Corresponds to 'type: string' in OpenAPI
|
||||
Map.class // Corresponds to 'type: object' in OpenAPI
|
||||
}
|
||||
)
|
||||
interface StringOrMapValue {}
|
||||
|
||||
@Valid
|
||||
private WorkerGroup workerGroup;
|
||||
|
||||
|
||||
@@ -24,10 +24,6 @@ public class PluginDefault {
|
||||
|
||||
@Schema(
|
||||
type = "object",
|
||||
oneOf = {
|
||||
Map.class,
|
||||
String.class
|
||||
},
|
||||
additionalProperties = Schema.AdditionalPropertiesValue.FALSE
|
||||
)
|
||||
private final Map<String, Object> values;
|
||||
|
||||
@@ -222,7 +222,6 @@ public class State {
|
||||
@Introspected
|
||||
public enum Type {
|
||||
CREATED,
|
||||
SUBMITTED,
|
||||
RUNNING,
|
||||
PAUSED,
|
||||
RESTARTED,
|
||||
|
||||
@@ -30,7 +30,7 @@ public class ResolvedTask {
|
||||
|
||||
public NextTaskRun toNextTaskRunIncrementIteration(Execution execution, Integer iteration) {
|
||||
return new NextTaskRun(
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 0),
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 1),
|
||||
this.getTask()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
public interface TaskInterface extends Plugin, PluginVersioning {
|
||||
@NotNull
|
||||
@@ -17,7 +19,7 @@ public interface TaskInterface extends Plugin, PluginVersioning {
|
||||
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
@Schema(title = "The class name of this task.")
|
||||
String getType();
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@Plugin
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@@ -22,7 +24,7 @@ public abstract class LogExporter<T extends Output> implements io.kestra.core.m
|
||||
protected String id;
|
||||
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
protected String type;
|
||||
|
||||
public abstract T sendLogs(RunContext runContext, Flux<LogRecord> logRecords) throws Exception;
|
||||
|
||||
@@ -8,12 +8,16 @@ public final class LogRecordMapper {
|
||||
private LogRecordMapper(){}
|
||||
|
||||
public static LogRecord mapToLogRecord(LogEntry log) {
|
||||
return mapToLogRecord(log, null);
|
||||
}
|
||||
|
||||
public static LogRecord mapToLogRecord(LogEntry log, Integer maxMessageSize) {
|
||||
return LogRecord.builder()
|
||||
.resource("Kestra")
|
||||
.timestampEpochNanos(instantInNanos(log.getTimestamp()))
|
||||
.severity(log.getLevel().name())
|
||||
.attributes(log.toLogMap())
|
||||
.bodyValue(LogEntry.toPrettyString(log))
|
||||
.bodyValue(LogEntry.toPrettyString(log, maxMessageSize))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static io.kestra.core.utils.WindowsUtils.windowsToUnixPath;
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
/**
|
||||
* Base class for all task runners.
|
||||
@@ -36,7 +37,7 @@ import static io.kestra.core.utils.WindowsUtils.windowsToUnixPath;
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
public abstract class TaskRunner<T extends TaskRunnerDetailResult> implements Plugin, PluginVersioning, WorkerJobLifecycle {
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
protected String type;
|
||||
|
||||
@PluginProperty(hidden = true, group = PluginProperty.CORE_GROUP)
|
||||
|
||||
@@ -47,7 +47,6 @@ abstract public class AbstractTrigger implements TriggerInterface {
|
||||
@Valid
|
||||
protected List<@Valid @NotNull Condition> conditions;
|
||||
|
||||
@NotNull
|
||||
@Builder.Default
|
||||
@PluginProperty(hidden = true, group = PluginProperty.CORE_GROUP)
|
||||
@Schema(defaultValue = "false")
|
||||
|
||||
@@ -7,6 +7,7 @@ import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
public interface TriggerInterface extends Plugin, PluginVersioning {
|
||||
@NotNull
|
||||
@@ -17,7 +18,7 @@ public interface TriggerInterface extends Plugin, PluginVersioning {
|
||||
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
@Schema(title = "The class name for this current trigger.")
|
||||
String getType();
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@io.kestra.core.models.annotations.Plugin
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@@ -15,6 +17,6 @@ import lombok.experimental.SuperBuilder;
|
||||
public abstract class AdditionalPlugin implements Plugin {
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
protected String type;
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
*
|
||||
* @return the {@link DefaultPluginRegistry}.
|
||||
*/
|
||||
public synchronized static DefaultPluginRegistry getOrCreate() {
|
||||
public static DefaultPluginRegistry getOrCreate() {
|
||||
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
|
||||
if (!instance.isInitialized()) {
|
||||
instance.init();
|
||||
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
/**
|
||||
* Initializes the registry by loading all core plugins.
|
||||
*/
|
||||
protected synchronized void init() {
|
||||
protected void init() {
|
||||
if (initialized.compareAndSet(false, true)) {
|
||||
register(scanner.scan());
|
||||
}
|
||||
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
if (existing != null && existing.crc32() == plugin.crc32()) {
|
||||
return; // same plugin already registered
|
||||
}
|
||||
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
if (existing != null) {
|
||||
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
|
||||
pluginClassByIdentifier.putAll(plugins);
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
|
||||
|
||||
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
|
||||
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::asText).orElse(null);
|
||||
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
|
||||
|
||||
if (type == null || type.isEmpty()) {
|
||||
return null;
|
||||
|
||||
@@ -11,7 +11,6 @@ import io.kestra.core.runners.*;
|
||||
|
||||
public interface QueueFactoryInterface {
|
||||
String EXECUTION_NAMED = "executionQueue";
|
||||
String EXECUTION_EVENT_NAMED = "executionEventQueue";
|
||||
String EXECUTOR_NAMED = "executorQueue";
|
||||
String WORKERJOB_NAMED = "workerJobQueue";
|
||||
String WORKERTASKRESULT_NAMED = "workerTaskResultQueue";
|
||||
@@ -31,8 +30,6 @@ public interface QueueFactoryInterface {
|
||||
|
||||
QueueInterface<Execution> execution();
|
||||
|
||||
QueueInterface<ExecutionEvent> executionEvent();
|
||||
|
||||
QueueInterface<Executor> executor();
|
||||
|
||||
WorkerJobQueueInterface workerJob();
|
||||
|
||||
@@ -35,24 +35,6 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
|
||||
|
||||
void delete(String consumerGroup, T message) throws QueueException;
|
||||
|
||||
/**
|
||||
* Delete all messages of the queue for this key.
|
||||
* This is used to purge a queue for a specific key.
|
||||
* A queue implementation may omit to implement it and purge records differently.
|
||||
*/
|
||||
default void deleteByKey(String key) throws QueueException {
|
||||
// by default do nothing
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all messages of the queue for a set of keys.
|
||||
* This is used to purge a queue for specific keys.
|
||||
* A queue implementation may omit to implement it and purge records differently.
|
||||
*/
|
||||
default void deleteByKeys(List<String> keys) throws QueueException {
|
||||
// by default do nothing
|
||||
}
|
||||
|
||||
default Runnable receive(Consumer<Either<T, DeserializationException>> consumer) {
|
||||
return receive(null, consumer, false);
|
||||
}
|
||||
@@ -72,20 +54,4 @@ public interface QueueInterface<T> extends Closeable, Pauseable {
|
||||
}
|
||||
|
||||
Runnable receive(String consumerGroup, Class<?> queueType, Consumer<Either<T, DeserializationException>> consumer, boolean forUpdate);
|
||||
|
||||
default Runnable receiveBatch(Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
|
||||
return receiveBatch(null, queueType, consumer);
|
||||
}
|
||||
|
||||
default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer) {
|
||||
return receiveBatch(consumerGroup, queueType, consumer, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumer a batch of messages.
|
||||
* By default, it consumes a single message, a queue implementation may implement it to support batch consumption.
|
||||
*/
|
||||
default Runnable receiveBatch(String consumerGroup, Class<?> queueType, Consumer<List<Either<T, DeserializationException>>> consumer, boolean forUpdate) {
|
||||
return receive(consumerGroup, either -> consumer.accept(List.of(either)), forUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,8 @@ public class QueueService {
|
||||
return ((SubflowExecution<?>) object).getExecution().getId();
|
||||
} else if (object.getClass() == SubflowExecutionResult.class) {
|
||||
return ((SubflowExecutionResult) object).getExecutionId();
|
||||
} else if (object.getClass() == ExecutorState.class) {
|
||||
return ((ExecutorState) object).getExecutionId();
|
||||
} else if (object.getClass() == Setting.class) {
|
||||
return ((Setting) object).getKey();
|
||||
} else if (object.getClass() == Executor.class) {
|
||||
|
||||
@@ -2,12 +2,12 @@ package io.kestra.core.repositories;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.executions.statistics.DailyExecutionStatistics;
|
||||
import io.kestra.core.models.executions.statistics.ExecutionCount;
|
||||
import io.kestra.core.models.executions.statistics.Flow;
|
||||
import io.kestra.core.models.flows.FlowScope;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.runners.Executor;
|
||||
import io.kestra.core.utils.DateUtils;
|
||||
import io.kestra.plugin.core.dashboard.data.Executions;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
@@ -25,6 +25,8 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
|
||||
Boolean isTaskRunEnabled();
|
||||
|
||||
default Optional<Execution> findById(String tenantId, String id) {
|
||||
return findById(tenantId, id, false);
|
||||
}
|
||||
@@ -94,6 +96,12 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
|
||||
Flux<Execution> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
ArrayListTotal<TaskRun> findTaskRun(
|
||||
Pageable pageable,
|
||||
@Nullable String tenantId,
|
||||
List<QueryFilter> filters
|
||||
);
|
||||
|
||||
Execution delete(Execution execution);
|
||||
|
||||
Integer purge(Execution execution);
|
||||
@@ -106,7 +114,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatistics(
|
||||
@@ -118,7 +127,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
List<State.Type> state
|
||||
List<State.Type> state,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
@Getter
|
||||
@@ -156,6 +166,4 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
String tenantId,
|
||||
@Nullable List<FlowFilter> flows
|
||||
);
|
||||
|
||||
Executor lock(String executionId, Function<Execution, Executor> function);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.plugin.core.dashboard.data.Flows;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -12,7 +11,7 @@ import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fields> {
|
||||
public interface FlowRepositoryInterface {
|
||||
|
||||
Optional<Flow> findById(String tenantId, String namespace, String id, Optional<Integer> revision, Boolean allowDeleted);
|
||||
|
||||
@@ -163,6 +162,4 @@ public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fie
|
||||
FlowWithSource update(GenericFlow flow, FlowInterface previous) throws ConstraintViolationException;
|
||||
|
||||
FlowWithSource delete(FlowInterface flow);
|
||||
|
||||
Boolean existAnyNoAcl(String tenantId);
|
||||
}
|
||||
|
||||
@@ -83,9 +83,7 @@ public class LocalFlowRepositoryLoader {
|
||||
}
|
||||
|
||||
public void load(String tenantId, File basePath) throws IOException {
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
|
||||
.stream()
|
||||
.filter(flow -> tenantId.equals(flow.getTenantId()))
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
|
||||
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
|
||||
|
||||
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {
|
||||
|
||||
@@ -5,7 +5,10 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.exceptions.InternalException;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
@@ -26,7 +29,6 @@ import org.apache.commons.lang3.stream.Streams;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.trace.Tracer.throwCallable;
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
@@ -151,22 +153,14 @@ public final class ExecutableUtils {
|
||||
currentFlow.getNamespace(),
|
||||
currentFlow.getId()
|
||||
)
|
||||
.orElseThrow(() -> {
|
||||
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
|
||||
runContext.logger().error(msg);
|
||||
return new IllegalStateException(msg);
|
||||
});
|
||||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
|
||||
|
||||
if (flow.isDisabled()) {
|
||||
String msg = "Cannot execute a flow which is disabled";
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute a flow which is disabled");
|
||||
}
|
||||
|
||||
if (flow instanceof FlowWithException fwe) {
|
||||
String msg = "Cannot execute an invalid flow: " + fwe.getException();
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
|
||||
}
|
||||
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
@@ -207,20 +201,7 @@ public final class ExecutableUtils {
|
||||
.build()
|
||||
)
|
||||
.withScheduleDate(scheduleOnDate);
|
||||
if(execution.getInputs().size()<inputs.size()) {
|
||||
Map<String,Object>resolvedInputs=execution.getInputs();
|
||||
for (var inputKey : inputs.keySet()) {
|
||||
if (!resolvedInputs.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
|
||||
inputKey,
|
||||
currentExecution.getId(),
|
||||
currentTask.subflowId().namespace(),
|
||||
currentTask.subflowId().flowId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inject the traceparent into the new execution
|
||||
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
public record ExecutionEvent(String tenantId, String executionId, Instant eventDate, ExecutionEventType eventType) implements HasUID {
|
||||
public ExecutionEvent(Execution execution, ExecutionEventType eventType) {
|
||||
this(execution.getTenantId(), execution.getId(), Instant.now(), eventType);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String uid() {
|
||||
return executionId;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
public enum ExecutionEventType {
|
||||
CREATED,
|
||||
UPDATED,
|
||||
TERMINATED,
|
||||
}
|
||||
21
core/src/main/java/io/kestra/core/runners/ExecutorState.java
Normal file
21
core/src/main/java/io/kestra/core/runners/ExecutorState.java
Normal file
@@ -0,0 +1,21 @@
|
||||
package io.kestra.core.runners;
|
||||
|
||||
import io.kestra.core.models.flows.State;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class ExecutorState {
|
||||
private String executionId;
|
||||
private Map<String, State.Type> workerTaskDeduplication = new ConcurrentHashMap<>();
|
||||
private Map<String, String> childDeduplication = new ConcurrentHashMap<>();
|
||||
private Map<String, State.Type> subflowExecutionDeduplication = new ConcurrentHashMap<>();
|
||||
|
||||
public ExecutorState(String executionId) {
|
||||
this.executionId = executionId;
|
||||
}
|
||||
}
|
||||
@@ -48,7 +48,15 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.util.*;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -220,19 +228,6 @@ public class FlowInputOutput {
|
||||
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
|
||||
})
|
||||
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||
if (resolved.size() < data.size()) {
|
||||
RunContext runContext = runContextFactory.of(flow, execution);
|
||||
for (var inputKey : data.keySet()) {
|
||||
if (!resolved.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
|
||||
inputKey,
|
||||
flow.getNamespace(),
|
||||
flow.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return MapUtils.flattenToNestedMap(resolved);
|
||||
}
|
||||
|
||||
@@ -287,9 +282,10 @@ public class FlowInputOutput {
|
||||
Input<?> input = resolvable.get().input();
|
||||
|
||||
try {
|
||||
// resolve all input dependencies and check whether input is enabled
|
||||
final Map<String, InputAndValue> dependencies = resolveAllDependentInputs(input, flow, execution, inputs, decryptSecrets);
|
||||
final RunContext runContext = buildRunContextForExecutionAndInputs(flow, execution, dependencies, decryptSecrets);
|
||||
// Resolve all input dependencies and check whether input is enabled
|
||||
// Note: Secrets are always decrypted here because they can be part of expressions used to render inputs such as SELECT & MULTI_SELECT.
|
||||
final Map<String, InputAndValue> dependencies = resolveAllDependentInputs(input, flow, execution, inputs, true);
|
||||
final RunContext runContext = buildRunContextForExecutionAndInputs(flow, execution, dependencies, true);
|
||||
|
||||
boolean isInputEnabled = dependencies.isEmpty() || dependencies.values().stream().allMatch(InputAndValue::enabled);
|
||||
|
||||
@@ -326,13 +322,14 @@ public class FlowInputOutput {
|
||||
resolvable.setInput(input);
|
||||
|
||||
Object value = resolvable.get().value();
|
||||
|
||||
|
||||
// resolve default if needed
|
||||
if (value == null && input.getDefaults() != null) {
|
||||
value = resolveDefaultValue(input, runContext);
|
||||
RunContext runContextForDefault = decryptSecrets ? runContext : buildRunContextForExecutionAndInputs(flow, execution, dependencies, false);
|
||||
value = resolveDefaultValue(input, runContextForDefault);
|
||||
resolvable.isDefault(true);
|
||||
}
|
||||
|
||||
|
||||
// validate and parse input value
|
||||
if (value == null) {
|
||||
if (input.getRequired()) {
|
||||
@@ -361,7 +358,7 @@ public class FlowInputOutput {
|
||||
|
||||
return resolvable.get();
|
||||
}
|
||||
|
||||
|
||||
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
|
||||
return switch (input.getType()) {
|
||||
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
|
||||
@@ -378,7 +375,7 @@ public class FlowInputOutput {
|
||||
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
|
||||
@@ -464,7 +461,7 @@ public class FlowInputOutput {
|
||||
if (data.getType() == null) {
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
|
||||
}
|
||||
|
||||
|
||||
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
|
||||
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(
|
||||
@@ -541,17 +538,17 @@ public class FlowInputOutput {
|
||||
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
if (outputs == null) return Map.of();
|
||||
|
||||
|
||||
// render required outputs
|
||||
Map<String, Object> outputsById = outputs
|
||||
.stream()
|
||||
.filter(output -> output.getRequired() == null || output.getRequired())
|
||||
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
|
||||
outputsById = runContext.render(outputsById);
|
||||
|
||||
|
||||
// render optional outputs one by one to catch, log, and skip any error.
|
||||
for (io.kestra.core.models.flows.Output output : outputs) {
|
||||
if (Boolean.FALSE.equals(output.getRequired())) {
|
||||
@@ -594,9 +591,9 @@ public class FlowInputOutput {
|
||||
}
|
||||
|
||||
public void isDefault(boolean isDefault) {
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
}
|
||||
|
||||
|
||||
public void setInput(final Input<?> input) {
|
||||
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
|
||||
}
|
||||
|
||||
@@ -84,12 +84,6 @@ public class FlowableUtils {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have submitted, leave
|
||||
Optional<TaskRun> lastSubmitted = execution.findLastSubmitted(taskRuns);
|
||||
if (lastSubmitted.isPresent()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have running, leave
|
||||
Optional<TaskRun> lastRunning = execution.findLastRunning(taskRuns);
|
||||
if (lastRunning.isPresent()) {
|
||||
|
||||
@@ -10,7 +10,6 @@ import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.Input;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.input.SecretInput;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.property.PropertyContext;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.models.triggers.AbstractTrigger;
|
||||
@@ -100,7 +99,7 @@ public final class RunVariables {
|
||||
* @return a new immutable {@link Map}.
|
||||
*/
|
||||
static Map<String, Object> of(final AbstractTrigger trigger) {
|
||||
return ImmutableMap.of(
|
||||
return Map.of(
|
||||
"id", trigger.getId(),
|
||||
"type", trigger.getType()
|
||||
);
|
||||
@@ -282,12 +281,15 @@ public final class RunVariables {
|
||||
}
|
||||
|
||||
if (flow != null && flow.getInputs() != null) {
|
||||
// Create a new PropertyContext with 'flow' variables which are required by some pebble expressions.
|
||||
PropertyContextWithVariables context = new PropertyContextWithVariables(propertyContext, Map.of("flow", RunVariables.of(flow)));
|
||||
|
||||
// we add default inputs value from the flow if not already set, this will be useful for triggers
|
||||
flow.getInputs().stream()
|
||||
.filter(input -> input.getDefaults() != null && !inputs.containsKey(input.getId()))
|
||||
.forEach(input -> {
|
||||
try {
|
||||
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, propertyContext));
|
||||
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, context));
|
||||
} catch (IllegalVariableEvaluationException e) {
|
||||
// Silent catch, if an input depends on another input, or a variable that is populated at runtime / input filling time, we can't resolve it here.
|
||||
}
|
||||
@@ -391,4 +393,20 @@ public final class RunVariables {
|
||||
}
|
||||
|
||||
private RunVariables(){}
|
||||
|
||||
private record PropertyContextWithVariables(
|
||||
PropertyContext delegate,
|
||||
Map<String, Object> variables
|
||||
) implements PropertyContext {
|
||||
|
||||
@Override
|
||||
public String render(String inline, Map<String, Object> variables) throws IllegalVariableEvaluationException {
|
||||
return delegate.render(inline, variables.isEmpty() ? this.variables : variables);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> render(Map<String, Object> inline, Map<String, Object> variables) throws IllegalVariableEvaluationException {
|
||||
return delegate.render(inline, variables.isEmpty() ? this.variables : variables);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,7 +168,6 @@ public class Extension extends AbstractExtension {
|
||||
functions.put("randomPort", new RandomPortFunction());
|
||||
functions.put("fileExists", fileExistsFunction);
|
||||
functions.put("isFileEmpty", isFileEmptyFunction);
|
||||
functions.put("nanoId", new NanoIDFunction());
|
||||
functions.put("tasksWithState", new TasksWithStateFunction());
|
||||
functions.put(HttpFunction.NAME, httpFunction);
|
||||
return functions;
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package io.kestra.core.runners.pebble.filters;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Filter;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ChunkFilter implements Filter {
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
@@ -30,6 +31,10 @@ public class ChunkFilter implements Filter {
|
||||
throw new PebbleException(null, "'chunk' filter can only be applied to List. Actual type was: " + input.getClass().getName(), lineNumber, self.getName());
|
||||
}
|
||||
|
||||
return Lists.partition((List) input, ((Long) args.get("size")).intValue());
|
||||
Object sizeObj = args.get("size");
|
||||
if (!(sizeObj instanceof Number)) {
|
||||
throw new PebbleException(null, "'chunk' filter argument 'size' must be a number. Actual type was: " + sizeObj.getClass().getName(), lineNumber, self.getName());
|
||||
}
|
||||
return Lists.partition((List) input, ((Number) sizeObj).intValue());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,12 +17,17 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class JqFilter implements Filter {
|
||||
private final Scope scope;
|
||||
// Load Scope once as static to avoid repeated initialization
|
||||
// This improves performance by loading builtin functions only once when the class loads
|
||||
private static final Scope SCOPE;
|
||||
private final List<String> argumentNames = new ArrayList<>();
|
||||
|
||||
static {
|
||||
SCOPE = Scope.newEmptyScope();
|
||||
BuiltinFunctionLoader.getInstance().loadFunctions(Versions.JQ_1_6, SCOPE);
|
||||
}
|
||||
|
||||
public JqFilter() {
|
||||
scope = Scope.newEmptyScope();
|
||||
BuiltinFunctionLoader.getInstance().loadFunctions(Versions.JQ_1_6, scope);
|
||||
this.argumentNames.add("expression");
|
||||
}
|
||||
|
||||
@@ -43,10 +48,7 @@ public class JqFilter implements Filter {
|
||||
|
||||
String pattern = (String) args.get("expression");
|
||||
|
||||
Scope rootScope = Scope.newEmptyScope();
|
||||
BuiltinFunctionLoader.getInstance().loadFunctions(Versions.JQ_1_6, rootScope);
|
||||
try {
|
||||
|
||||
JsonQuery q = JsonQuery.compile(pattern, Versions.JQ_1_6);
|
||||
|
||||
JsonNode in;
|
||||
@@ -59,7 +61,7 @@ public class JqFilter implements Filter {
|
||||
final List<Object> out = new ArrayList<>();
|
||||
|
||||
try {
|
||||
q.apply(scope, in, v -> {
|
||||
q.apply(Scope.newChildScope(SCOPE), in, v -> {
|
||||
if (v instanceof TextNode) {
|
||||
out.add(v.textValue());
|
||||
} else if (v instanceof NullNode) {
|
||||
|
||||
@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
|
||||
ZoneId zoneId = zoneId(timeZone);
|
||||
ZonedDateTime date = convert(input, zoneId, existingFormat);
|
||||
|
||||
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import io.kestra.core.http.HttpRequest;
|
||||
import io.kestra.core.http.HttpResponse;
|
||||
import io.kestra.core.http.client.HttpClient;
|
||||
import io.kestra.core.http.client.HttpClientException;
|
||||
import io.kestra.core.http.client.HttpClientRequestException;
|
||||
import io.kestra.core.http.client.HttpClientResponseException;
|
||||
import io.kestra.core.http.client.configurations.HttpConfiguration;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
@@ -103,15 +101,8 @@ public class HttpFunction<T> implements Function {
|
||||
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
|
||||
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
|
||||
return response.getBody();
|
||||
} catch (HttpClientResponseException e) {
|
||||
if (e.getResponse() != null) {
|
||||
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
|
||||
throw new PebbleException(e, msg , lineNumber, self.getName());
|
||||
} else {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
|
||||
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ public class KvFunction implements Function {
|
||||
String key = getKey(args, self, lineNumber);
|
||||
String namespace = (String) args.get(NAMESPACE_ARG);
|
||||
|
||||
Boolean errorOnMissing = Optional.ofNullable((Boolean) args.get(ERROR_ON_MISSING_ARG)).orElse(true);
|
||||
boolean errorOnMissing = Optional.ofNullable((Boolean) args.get(ERROR_ON_MISSING_ARG)).orElse(true);
|
||||
|
||||
Map<String, String> flow = (Map<String, String>) context.getVariable("flow");
|
||||
String flowNamespace = flow.get(NAMESPACE_ARG);
|
||||
@@ -53,11 +53,16 @@ public class KvFunction implements Function {
|
||||
// we didn't check allowedNamespace here as it's checked in the kvStoreService itself
|
||||
value = kvStoreService.get(flowTenantId, namespace, flowNamespace).getValue(key);
|
||||
}
|
||||
} catch (ResourceExpiredException e) {
|
||||
if (errorOnMissing) {
|
||||
throw new PebbleException(e, e.getMessage(), lineNumber, self.getName());
|
||||
}
|
||||
value = Optional.empty();
|
||||
} catch (Exception e) {
|
||||
throw new PebbleException(e, e.getMessage(), lineNumber, self.getName());
|
||||
}
|
||||
|
||||
if (value.isEmpty() && errorOnMissing == Boolean.TRUE) {
|
||||
if (value.isEmpty() && errorOnMissing) {
|
||||
throw new PebbleException(null, "The key '" + key + "' does not exist in the namespace '" + namespace + "'.", lineNumber, self.getName());
|
||||
}
|
||||
|
||||
@@ -85,4 +90,4 @@ public class KvFunction implements Function {
|
||||
|
||||
return (String) args.get(KEY_ARGS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package io.kestra.core.runners.pebble.functions;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Function;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NanoIDFunction implements Function {
|
||||
|
||||
private static final int DEFAULT_LENGTH = 21;
|
||||
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
private static final String LENGTH = "length";
|
||||
private static final String ALPHABET = "alphabet";
|
||||
|
||||
private static final int MAX_LENGTH = 1000;
|
||||
|
||||
@Override
|
||||
public Object execute(
|
||||
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
|
||||
int length = DEFAULT_LENGTH;
|
||||
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
|
||||
length = parseLength(args, self, lineNumber);
|
||||
}
|
||||
char[] alphabet = DEFAULT_ALPHABET;
|
||||
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
|
||||
alphabet = ((String) args.get(ALPHABET)).toCharArray();
|
||||
}
|
||||
return createNanoID(length, alphabet);
|
||||
}
|
||||
|
||||
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
|
||||
var value = (Long) args.get(LENGTH);
|
||||
if(value > MAX_LENGTH) {
|
||||
throw new PebbleException(
|
||||
null,
|
||||
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
|
||||
lineNumber,
|
||||
self.getName());
|
||||
}
|
||||
return Math.toIntExact(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
return List.of(LENGTH,ALPHABET);
|
||||
}
|
||||
|
||||
String createNanoID(int length, char[] alphabet){
|
||||
final char[] data = new char[length];
|
||||
final byte[] bytes = new byte[length];
|
||||
final int mask = alphabet.length-1;
|
||||
secureRandom.nextBytes(bytes);
|
||||
for (int i = 0; i < length; ++i) {
|
||||
data[i] = alphabet[bytes[i] & mask];
|
||||
}
|
||||
return String.valueOf(data);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -180,13 +180,23 @@ public final class FileSerde {
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.flows.input.InputAndValue;
|
||||
import io.kestra.core.models.hierarchies.AbstractGraphTask;
|
||||
import io.kestra.core.models.hierarchies.GraphCluster;
|
||||
import io.kestra.core.models.tasks.FlowableTask;
|
||||
import io.kestra.core.models.tasks.ResolvedTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
import io.kestra.core.models.tasks.retrys.AbstractRetry;
|
||||
@@ -121,21 +122,38 @@ public class ExecutionService {
|
||||
* Retry set the given taskRun in created state
|
||||
* and return the execution in running state
|
||||
**/
|
||||
public Execution retryTask(Execution execution, String taskRunId) {
|
||||
List<TaskRun> newTaskRuns = execution
|
||||
.getTaskRunList()
|
||||
.stream()
|
||||
.map(taskRun -> {
|
||||
if (taskRun.getId().equals(taskRunId)) {
|
||||
return taskRun
|
||||
.withState(State.Type.CREATED);
|
||||
public Execution retryTask(Execution execution, Flow flow, String taskRunId) throws InternalException {
|
||||
TaskRun taskRun = execution.findTaskRunByTaskRunId(taskRunId).withState(State.Type.CREATED);
|
||||
List<TaskRun> taskRunList = execution.getTaskRunList();
|
||||
|
||||
if (taskRun.getParentTaskRunId() != null) {
|
||||
// we need to find the parent to remove any errors or finally tasks already executed
|
||||
TaskRun parentTaskRun = execution.findTaskRunByTaskRunId(taskRun.getParentTaskRunId());
|
||||
Task parentTask = flow.findTaskByTaskId(parentTaskRun.getTaskId());
|
||||
if (parentTask instanceof FlowableTask<?> flowableTask) {
|
||||
if (flowableTask.getErrors() != null) {
|
||||
List<Task> allErrors = Stream.concat(flowableTask.getErrors().stream()
|
||||
.filter(task -> task.isFlowable() && ((FlowableTask<?>) task).getErrors() != null)
|
||||
.flatMap(task -> ((FlowableTask<?>) task).getErrors().stream()),
|
||||
flowableTask.getErrors().stream())
|
||||
.toList();
|
||||
allErrors.forEach(error -> taskRunList.removeIf(t -> t.getTaskId().equals(error.getId())));
|
||||
}
|
||||
|
||||
return taskRun;
|
||||
})
|
||||
.toList();
|
||||
if (flowableTask.getFinally() != null) {
|
||||
List<Task> allFinally = Stream.concat(flowableTask.getFinally().stream()
|
||||
.filter(task -> task.isFlowable() && ((FlowableTask<?>) task).getFinally() != null)
|
||||
.flatMap(task -> ((FlowableTask<?>) task).getFinally().stream()),
|
||||
flowableTask.getFinally().stream())
|
||||
.toList();
|
||||
allFinally.forEach(error -> taskRunList.removeIf(t -> t.getTaskId().equals(error.getId())));
|
||||
}
|
||||
}
|
||||
|
||||
return execution.withTaskRunList(newTaskRuns).withState(State.Type.RUNNING);
|
||||
return execution.withTaskRunList(taskRunList).withTaskRun(taskRun).withState(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
return execution.withTaskRun(taskRun).withState(State.Type.RUNNING);
|
||||
}
|
||||
|
||||
public Execution retryWaitFor(Execution execution, String flowableTaskRunId) {
|
||||
@@ -709,7 +727,7 @@ public class ExecutionService {
|
||||
// An edge case can exist where the execution is resumed automatically before we resume it with a killing.
|
||||
try {
|
||||
newExecution = this.resume(execution, flow, State.Type.KILLING, null);
|
||||
newExecution = newExecution.withState(afterKillState.orElse(newExecution.getState().getCurrent()));
|
||||
newExecution = newExecution.withState(killingOrAfterKillState);
|
||||
} catch (Exception e) {
|
||||
// if we cannot resume, we set it anyway to killing, so we don't throw
|
||||
log.warn("Unable to resume a paused execution before killing it", e);
|
||||
@@ -718,11 +736,12 @@ public class ExecutionService {
|
||||
} else {
|
||||
newExecution = execution.withState(killingOrAfterKillState);
|
||||
}
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
// immediately without publishing a CrudEvent like it's done on pause/resume method.
|
||||
return newExecution;
|
||||
}
|
||||
|
||||
public Execution kill(Execution execution, FlowInterface flow) {
|
||||
return this.kill(execution, flow, Optional.empty());
|
||||
}
|
||||
|
||||
@@ -5,14 +5,11 @@ import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class SkipExecutionService {
|
||||
private volatile List<String> skipExecutions = Collections.emptyList();
|
||||
private volatile List<FlowId> skipFlows = Collections.emptyList();
|
||||
@@ -25,11 +22,11 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
public synchronized void setSkipFlows(List<String> skipFlows) {
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).filter(Objects::nonNull).toList();
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipNamespaces(List<String> skipNamespaces) {
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).filter(Objects::nonNull).toList();
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipTenants(List<String> skipTenants) {
|
||||
@@ -76,31 +73,19 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
record FlowId(String tenant, String namespace, String flow) {
|
||||
static @Nullable FlowId from(String flowId) {
|
||||
static FlowId from(String flowId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(flowId);
|
||||
if (parts.length == 3) {
|
||||
return new FlowId(parts[0], parts[1], parts[2]);
|
||||
} else if (parts.length == 2) {
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid flow skip with values: '{}'", flowId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
|
||||
record NamespaceId(String tenant, String namespace) {
|
||||
static @Nullable NamespaceId from(String namespaceId) {
|
||||
static NamespaceId from(String namespaceId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(namespaceId);
|
||||
|
||||
if (parts.length == 2) {
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid namespace skip with values:'{}'", namespaceId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,12 +14,8 @@ import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
@@ -36,11 +32,7 @@ public abstract class StorageService {
|
||||
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(runContext.storage().getFile(from)))) {
|
||||
List<Path> splited;
|
||||
|
||||
if (storageSplitInterface.getRegexPattern() != null) {
|
||||
String renderedPattern = runContext.render(storageSplitInterface.getRegexPattern()).as(String.class).orElseThrow();
|
||||
String separator = runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow();
|
||||
splited = StorageService.splitByRegex(runContext, extension, separator, bufferedReader, renderedPattern);
|
||||
} else if (storageSplitInterface.getBytes() != null) {
|
||||
if (storageSplitInterface.getBytes() != null) {
|
||||
ReadableBytesTypeConverter readableBytesTypeConverter = new ReadableBytesTypeConverter();
|
||||
Number convert = readableBytesTypeConverter.convert(runContext.render(storageSplitInterface.getBytes()).as(String.class).orElseThrow(), Number.class)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Invalid size with value '" + storageSplitInterface.getBytes() + "'"));
|
||||
@@ -55,7 +47,7 @@ public abstract class StorageService {
|
||||
splited = StorageService.split(runContext, extension, runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow(),
|
||||
bufferedReader, (bytes, size) -> size >= renderedRows);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, rows, nor regexPattern");
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, nor rows");
|
||||
}
|
||||
|
||||
return splited
|
||||
@@ -125,36 +117,4 @@ public abstract class StorageService {
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
private static List<Path> splitByRegex(RunContext runContext, String extension, String separator, BufferedReader bufferedReader, String regexPattern) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
Map<String, RandomAccessFile> writers = new HashMap<>();
|
||||
Pattern pattern = Pattern.compile(regexPattern);
|
||||
|
||||
String row;
|
||||
while ((row = bufferedReader.readLine()) != null) {
|
||||
Matcher matcher = pattern.matcher(row);
|
||||
|
||||
if (matcher.find() && matcher.groupCount() > 0) {
|
||||
String routingKey = matcher.group(1);
|
||||
|
||||
// Get or create writer for this routing key
|
||||
RandomAccessFile writer = writers.get(routingKey);
|
||||
if (writer == null) {
|
||||
Path path = runContext.workingDir().createTempFile(extension);
|
||||
files.add(path);
|
||||
writer = new RandomAccessFile(path.toFile(), "rw");
|
||||
writers.put(routingKey, writer);
|
||||
}
|
||||
|
||||
byte[] bytes = (row + separator).getBytes(StandardCharsets.UTF_8);
|
||||
writer.getChannel().write(ByteBuffer.wrap(bytes));
|
||||
}
|
||||
// Lines that don't match the pattern are ignored
|
||||
}
|
||||
|
||||
writers.values().forEach(throwConsumer(RandomAccessFile::close));
|
||||
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -26,13 +26,4 @@ public interface StorageSplitInterface {
|
||||
defaultValue = "\\n"
|
||||
)
|
||||
Property<String> getSeparator();
|
||||
|
||||
@Schema(
|
||||
title = "Split file by regex pattern with the first capture group as the routing key.",
|
||||
description = "A regular expression pattern with a capture group. Lines matching this pattern will be grouped by the captured value. " +
|
||||
"For example, `\\[(\\w+)\\]` will group lines by log level (ERROR, WARN, INFO) extracted from log entries. " +
|
||||
"Lines with the same captured value will be written to the same output file. " +
|
||||
"This allows content-based splitting where the file is divided based on data patterns rather than size or line count."
|
||||
)
|
||||
Property<String> getRegexPattern();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
package io.kestra.core.utils;
|
||||
|
||||
public class RegexPatterns {
|
||||
public static final String JAVA_IDENTIFIER_REGEX = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$";
|
||||
}
|
||||
@@ -6,10 +6,10 @@ abstract public class TruthUtils {
|
||||
private static final List<String> FALSE_VALUES = List.of("false", "0", "-0", "");
|
||||
|
||||
public static boolean isTruthy(String condition) {
|
||||
return condition != null && !FALSE_VALUES.contains(condition.trim());
|
||||
return condition != null && !FALSE_VALUES.contains(condition);
|
||||
}
|
||||
|
||||
public static boolean isFalsy(String condition) {
|
||||
return condition != null && FALSE_VALUES.contains(condition.trim());
|
||||
return condition != null && FALSE_VALUES.contains(condition);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,84 +32,48 @@ public class Version implements Comparable<Version> {
|
||||
* @param version the version.
|
||||
* @return a new {@link Version} instance.
|
||||
*/
|
||||
public static Version of(final Object version) {
|
||||
public static Version of(String version) {
|
||||
|
||||
if (Objects.isNull(version)) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse null version");
|
||||
}
|
||||
|
||||
String strVersion = version.toString();
|
||||
|
||||
if (strVersion.startsWith("v")) {
|
||||
strVersion = strVersion.substring(1);
|
||||
if (version.startsWith("v")) {
|
||||
version = version.substring(1);
|
||||
}
|
||||
|
||||
int qualifier = strVersion.indexOf("-");
|
||||
int qualifier = version.indexOf("-");
|
||||
|
||||
final String[] versions = qualifier > 0 ?
|
||||
strVersion.substring(0, qualifier).split("\\.") :
|
||||
strVersion.split("\\.");
|
||||
version.substring(0, qualifier).split("\\.") :
|
||||
version.split("\\.");
|
||||
try {
|
||||
final int majorVersion = Integer.parseInt(versions[0]);
|
||||
final Integer minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : null;
|
||||
final Integer incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : null;
|
||||
final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0;
|
||||
final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0;
|
||||
|
||||
return new Version(
|
||||
majorVersion,
|
||||
minorVersion,
|
||||
incrementalVersion,
|
||||
qualifier > 0 ? strVersion.substring(qualifier + 1) : null,
|
||||
strVersion
|
||||
qualifier > 0 ? version.substring(qualifier + 1) : null,
|
||||
version
|
||||
);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resolves the most appropriate stable version from a collection, based on a given input version.
|
||||
* <p>
|
||||
* The matching rules are:
|
||||
* <ul>
|
||||
* <li>If {@code from} specifies only a major version (e.g. {@code 1}), return the latest stable version
|
||||
* with the same major (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a major and minor version only (e.g. {@code 1.2}), return the latest
|
||||
* stable version with the same major and minor (e.g. {@code 1.2.3}).</li>
|
||||
* <li>If {@code from} specifies a full version with major, minor, and patch (e.g. {@code 1.2.2}),
|
||||
* then only return it if it is exactly present (and stable) in {@code versions}.
|
||||
* No "upgrade" is performed in this case.</li>
|
||||
* <li>If no suitable version is found, returns {@code null}.</li>
|
||||
* </ul>
|
||||
* Static helper method for returning the most recent stable version for a current {@link Version}.
|
||||
*
|
||||
* @param from the reference version (may specify only major, or major+minor, or major+minor+patch).
|
||||
* @param versions the collection of candidate versions to resolve against.
|
||||
* @return the best matching stable version, or {@code null} if none match.
|
||||
* @param from the current version.
|
||||
* @param versions the list of version.
|
||||
*
|
||||
* @return the last stable version.
|
||||
*/
|
||||
public static Version getStable(final Version from, final Collection<Version> versions) {
|
||||
// Case 1: "from" is only a major (e.g. 1)
|
||||
if (from.hasOnlyMajor()) {
|
||||
List<Version> sameMajor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion())
|
||||
.toList();
|
||||
return sameMajor.isEmpty() ? null : Version.getLatest(sameMajor);
|
||||
}
|
||||
|
||||
// Case 2: "from" is major+minor only (e.g. 1.2)
|
||||
if (from.hasMajorAndMinorOnly()) {
|
||||
List<Version> sameMinor = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion()
|
||||
&& v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
return sameMinor.isEmpty() ? null : Version.getLatest(sameMinor);
|
||||
}
|
||||
|
||||
// Case 3: "from" is full version (major+minor+patch)
|
||||
if (versions.contains(from)) {
|
||||
return from;
|
||||
}
|
||||
|
||||
// No match
|
||||
return null;
|
||||
List<Version> compatibleVersions = versions.stream()
|
||||
.filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion())
|
||||
.toList();
|
||||
if (compatibleVersions.isEmpty()) return null;
|
||||
return Version.getLatest(compatibleVersions);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -159,8 +123,8 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
private final int majorVersion;
|
||||
private final Integer minorVersion;
|
||||
private final Integer patchVersion;
|
||||
private final int minorVersion;
|
||||
private final int incrementalVersion;
|
||||
private final Qualifier qualifier;
|
||||
|
||||
private final String originalVersion;
|
||||
@@ -170,14 +134,14 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
*/
|
||||
public Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int patchVersion,
|
||||
final int incrementalVersion,
|
||||
final String qualifier) {
|
||||
this(majorVersion, minorVersion, patchVersion, qualifier, null);
|
||||
this(majorVersion, minorVersion, incrementalVersion, qualifier, null);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -185,25 +149,25 @@ public class Version implements Comparable<Version> {
|
||||
*
|
||||
* @param majorVersion the major version (must be superior or equal to 0).
|
||||
* @param minorVersion the minor version (must be superior or equal to 0).
|
||||
* @param patchVersion the incremental version (must be superior or equal to 0).
|
||||
* @param incrementalVersion the incremental version (must be superior or equal to 0).
|
||||
* @param qualifier the qualifier.
|
||||
* @param originalVersion the original string version.
|
||||
*/
|
||||
private Version(final Integer majorVersion,
|
||||
final Integer minorVersion,
|
||||
final Integer patchVersion,
|
||||
private Version(final int majorVersion,
|
||||
final int minorVersion,
|
||||
final int incrementalVersion,
|
||||
final String qualifier,
|
||||
final String originalVersion) {
|
||||
this.majorVersion = requirePositive(majorVersion, "major");
|
||||
this.minorVersion = requirePositive(minorVersion, "minor");
|
||||
this.patchVersion = requirePositive(patchVersion, "incremental");
|
||||
this.incrementalVersion = requirePositive(incrementalVersion, "incremental");
|
||||
this.qualifier = qualifier != null ? new Qualifier(qualifier) : null;
|
||||
this.originalVersion = originalVersion;
|
||||
}
|
||||
|
||||
|
||||
private static Integer requirePositive(Integer version, final String message) {
|
||||
if (version != null && version < 0) {
|
||||
private static int requirePositive(int version, final String message) {
|
||||
if (version < 0) {
|
||||
throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message));
|
||||
}
|
||||
return version;
|
||||
@@ -214,11 +178,11 @@ public class Version implements Comparable<Version> {
|
||||
}
|
||||
|
||||
public int minorVersion() {
|
||||
return minorVersion != null ? minorVersion : 0;
|
||||
return minorVersion;
|
||||
}
|
||||
|
||||
public int patchVersion() {
|
||||
return patchVersion != null ? patchVersion : 0;
|
||||
public int incrementalVersion() {
|
||||
return incrementalVersion;
|
||||
}
|
||||
|
||||
public Qualifier qualifier() {
|
||||
@@ -233,9 +197,9 @@ public class Version implements Comparable<Version> {
|
||||
if (this == o) return true;
|
||||
if (!(o instanceof Version)) return false;
|
||||
Version version = (Version) o;
|
||||
return Objects.equals(majorVersion,version.majorVersion) &&
|
||||
Objects.equals(minorVersion, version.minorVersion) &&
|
||||
Objects.equals(patchVersion,version.patchVersion) &&
|
||||
return majorVersion == version.majorVersion &&
|
||||
minorVersion == version.minorVersion &&
|
||||
incrementalVersion == version.incrementalVersion &&
|
||||
Objects.equals(qualifier, version.qualifier);
|
||||
}
|
||||
|
||||
@@ -244,7 +208,7 @@ public class Version implements Comparable<Version> {
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(majorVersion, minorVersion, patchVersion, qualifier);
|
||||
return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -254,7 +218,7 @@ public class Version implements Comparable<Version> {
|
||||
public String toString() {
|
||||
if (originalVersion != null) return originalVersion;
|
||||
|
||||
String version = majorVersion + "." + minorVersion + "." + patchVersion;
|
||||
String version = majorVersion + "." + minorVersion + "." + incrementalVersion;
|
||||
return (qualifier != null) ? version +"-" + qualifier : version;
|
||||
}
|
||||
|
||||
@@ -274,7 +238,7 @@ public class Version implements Comparable<Version> {
|
||||
return compareMinor;
|
||||
}
|
||||
|
||||
int compareIncremental = Integer.compare(that.patchVersion, this.patchVersion);
|
||||
int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion);
|
||||
if (compareIncremental != 0) {
|
||||
return compareIncremental;
|
||||
}
|
||||
@@ -289,21 +253,6 @@ public class Version implements Comparable<Version> {
|
||||
|
||||
return this.qualifier.compareTo(that.qualifier);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true if only major is specified (e.g. "1")
|
||||
*/
|
||||
private boolean hasOnlyMajor() {
|
||||
return minorVersion == null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if major+minor are specified, but no patch (e.g. "1.2")
|
||||
*/
|
||||
private boolean hasMajorAndMinorOnly() {
|
||||
return minorVersion != null && patchVersion == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether this version is before the given one.
|
||||
|
||||
@@ -46,19 +46,16 @@ public class VersionProvider {
|
||||
this.date = loadTime(gitProperties);
|
||||
this.version = loadVersion(buildProperties, gitProperties);
|
||||
|
||||
// check the version in the settings and update if needed, we didn't use it would allow us to detect incompatible update later if needed
|
||||
settingRepository.ifPresent(
|
||||
settingRepositoryInterface -> persistVersion(settingRepositoryInterface, version));
|
||||
}
|
||||
|
||||
private static synchronized void persistVersion(SettingRepositoryInterface settingRepositoryInterface, String version) {
|
||||
Optional<Setting> versionSetting = settingRepositoryInterface.findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(version)) {
|
||||
settingRepositoryInterface.save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(version)
|
||||
.build()
|
||||
);
|
||||
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
|
||||
if (settingRepository.isPresent()) {
|
||||
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
|
||||
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
|
||||
settingRepository.get().save(Setting.builder()
|
||||
.key(Setting.INSTANCE_VERSION)
|
||||
.value(this.version)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.Getter;
|
||||
|
||||
import static io.kestra.core.utils.RegexPatterns.JAVA_IDENTIFIER_REGEX;
|
||||
|
||||
@Getter
|
||||
@JsonTypeInfo(
|
||||
use = JsonTypeInfo.Id.NAME,
|
||||
@@ -20,6 +22,6 @@ import lombok.Getter;
|
||||
public class MarkdownSource {
|
||||
@NotNull
|
||||
@NotBlank
|
||||
@Pattern(regexp = "^[A-Za-z_$][A-Za-z0-9_$]*(\\.[A-Za-z_$][A-Za-z0-9_$]*)*$")
|
||||
@Pattern(regexp = JAVA_IDENTIFIER_REGEX)
|
||||
private String type;
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilter;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display Flow data in a dashboard chart."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display a chart with a list of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: list_flows
|
||||
type: io.kestra.plugin.core.dashboard.chart.Table
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.Flows
|
||||
columns:
|
||||
namespace:
|
||||
field: NAMESPACE
|
||||
id:
|
||||
field: ID
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("Flows")
|
||||
public class Flows<C extends ColumnDescriptor<Flows.Fields>> extends DataFilter<Flows.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeName;
|
||||
import io.kestra.core.models.annotations.Example;
|
||||
import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.ColumnDescriptor;
|
||||
import io.kestra.core.models.dashboards.DataFilterKPI;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@SuperBuilder(toBuilder = true)
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
@EqualsAndHashCode
|
||||
@Schema(
|
||||
title = "Display a chart with Flows KPI.",
|
||||
description = "Change."
|
||||
)
|
||||
@Plugin(
|
||||
examples = {
|
||||
@Example(
|
||||
title = "Display count of Flows.",
|
||||
full = true,
|
||||
code = { """
|
||||
charts:
|
||||
- id: kpi
|
||||
type: io.kestra.plugin.core.dashboard.chart.KPI
|
||||
data:
|
||||
type: io.kestra.plugin.core.dashboard.data.FlowsKPI
|
||||
columns:
|
||||
field: ID
|
||||
agg: COUNT
|
||||
"""
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
@JsonTypeName("FlowsKPI")
|
||||
public class FlowsKPI<C extends ColumnDescriptor<FlowsKPI.Fields>> extends DataFilterKPI<FlowsKPI.Fields, C> implements IFlows {
|
||||
@Override
|
||||
public Class<? extends QueryBuilderInterface<Fields>> repositoryClass() {
|
||||
return FlowRepositoryInterface.class;
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package io.kestra.plugin.core.dashboard.data;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.dashboards.filters.AbstractFilter;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public interface IFlows extends IData<IFlows.Fields> {
|
||||
|
||||
default List<AbstractFilter<IFlows.Fields>> whereWithGlobalFilters(List<QueryFilter> filters, ZonedDateTime startDate, ZonedDateTime endDate, List<AbstractFilter<IFlows.Fields>> where) {
|
||||
List<AbstractFilter<IFlows.Fields>> updatedWhere = where != null ? new ArrayList<>(where) : new ArrayList<>();
|
||||
|
||||
if (ListUtils.isEmpty(filters)) {
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
List<QueryFilter> namespaceFilters = filters.stream().filter(f -> f.field().equals(QueryFilter.Field.NAMESPACE)).toList();
|
||||
if (!namespaceFilters.isEmpty()) {
|
||||
updatedWhere.removeIf(filter -> filter.getField().equals(IFlows.Fields.NAMESPACE));
|
||||
namespaceFilters.forEach(f -> {
|
||||
updatedWhere.add(f.toDashboardFilterBuilder(IFlows.Fields.NAMESPACE, f.value()));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
return updatedWhere;
|
||||
}
|
||||
|
||||
enum Fields {
|
||||
ID,
|
||||
NAMESPACE,
|
||||
REVISION
|
||||
}
|
||||
}
|
||||
@@ -127,24 +127,9 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
}
|
||||
|
||||
// check for system labels: none can be passed at runtime
|
||||
Optional<Map.Entry<String, String>> systemLabel = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX))
|
||||
.findFirst();
|
||||
if (systemLabel.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"System labels can only be set by Kestra itself, offending label: " +
|
||||
systemLabel.get().getKey() + "=" + systemLabel.get().getValue()
|
||||
);
|
||||
}
|
||||
|
||||
// check for empty label values
|
||||
Optional<Map.Entry<String, String>> emptyValue = labelsAsMap.entrySet().stream()
|
||||
.filter(entry -> entry.getValue().isEmpty())
|
||||
.findFirst();
|
||||
if (emptyValue.isPresent()) {
|
||||
throw new IllegalArgumentException(
|
||||
"Label values cannot be empty, offending label: " + emptyValue.get().getKey()
|
||||
);
|
||||
Optional<Map.Entry<String, String>> first = labelsAsMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX)).findFirst();
|
||||
if (first.isPresent()) {
|
||||
throw new IllegalArgumentException("System labels can only be set by Kestra itself, offending label: " + first.get().getKey() + "=" + first.get().getValue());
|
||||
}
|
||||
|
||||
Map<String, String> newLabels = ListUtils.emptyOnNull(execution.getLabels()).stream()
|
||||
@@ -155,7 +140,6 @@ public class Labels extends Task implements ExecutionUpdatableTask {
|
||||
newLabels.putAll(labelsAsMap);
|
||||
|
||||
return execution.withLabels(newLabels.entrySet().stream()
|
||||
.filter(Label.getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(
|
||||
entry.getKey(),
|
||||
entry.getValue()
|
||||
|
||||
@@ -478,7 +478,7 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
try (InputStream is = runContext.storage().getFile(splitsURI)){
|
||||
String fileContent = new String(is.readAllBytes());
|
||||
List<URI> splits = fileContent.lines().map(line -> URI.create(line)).toList();
|
||||
AtomicInteger currentIteration = new AtomicInteger(0);
|
||||
AtomicInteger currentIteration = new AtomicInteger(1);
|
||||
|
||||
return splits
|
||||
.stream()
|
||||
@@ -648,8 +648,6 @@ public class ForEachItem extends Task implements FlowableTask<VoidOutput>, Child
|
||||
|
||||
@Builder.Default
|
||||
private Property<String> separator = Property.ofValue("\n");
|
||||
|
||||
private Property<String> regexPattern;
|
||||
}
|
||||
|
||||
@Builder
|
||||
|
||||
@@ -102,7 +102,7 @@ public class Switch extends Task implements FlowableTask<Switch.Output> {
|
||||
@Schema(
|
||||
title = "The map of keys and a list of tasks to be executed if the conditional `value` matches the key"
|
||||
)
|
||||
@PluginProperty
|
||||
@PluginProperty(additionalProperties = Task[].class)
|
||||
private Map<String, List<Task>> cases;
|
||||
|
||||
@Valid
|
||||
|
||||
@@ -48,13 +48,6 @@ import java.util.List;
|
||||
"partitions: 8"
|
||||
}
|
||||
),
|
||||
@Example(
|
||||
title = "Split a file by regex pattern - group lines by captured value.",
|
||||
code = {
|
||||
"from: \"kestra://long/url/logs.txt\"",
|
||||
"regexPattern: \"\\\\[(\\\\w+)\\\\]\""
|
||||
}
|
||||
),
|
||||
},
|
||||
aliases = "io.kestra.core.tasks.storages.Split"
|
||||
)
|
||||
@@ -72,13 +65,6 @@ public class Split extends Task implements RunnableTask<Split.Output>, StorageSp
|
||||
|
||||
private Property<Integer> rows;
|
||||
|
||||
@Schema(
|
||||
title = "Split file by regex pattern. Lines are grouped by the first capture group value.",
|
||||
description = "A regular expression pattern with a capture group. Lines matching this pattern will be grouped by the captured value. For example, `\\[(\\w+)\\]` will group lines by log level (ERROR, WARN, INFO) extracted from log entries."
|
||||
)
|
||||
@PluginProperty(dynamic = true)
|
||||
private Property<String> regexPattern;
|
||||
|
||||
@Builder.Default
|
||||
private Property<String> separator = Property.ofValue("\n");
|
||||
|
||||
|
||||
@@ -48,38 +48,6 @@ import jakarta.validation.constraints.Size;
|
||||
- 200 if the webhook triggers an execution.
|
||||
- 204 if the webhook cannot trigger an execution due to a lack of matching event conditions sent by other application.
|
||||
|
||||
The response body will contain the execution ID if the execution is successfully triggered using the following format:
|
||||
```json
|
||||
{
|
||||
"tenantId": "your_tenant_id",
|
||||
"namespace": "your_namespace",
|
||||
"flowId": "your_flow_id",
|
||||
"flowRevision": 1,
|
||||
"trigger": {
|
||||
"id": "the_trigger_id",
|
||||
"type": "io.kestra.plugin.core.trigger.Webhook",
|
||||
"variables": {
|
||||
# The variables sent by the webhook caller
|
||||
},
|
||||
"logFile": "the_log_file_url"
|
||||
},
|
||||
"outputs": {
|
||||
# The outputs of the flow, only available if `wait` is set to true
|
||||
},
|
||||
"labels": [
|
||||
{"key": "value" }
|
||||
],
|
||||
"state": {
|
||||
"type": "RUNNING",
|
||||
"histories": [
|
||||
# The state histories of the execution
|
||||
]
|
||||
},
|
||||
"url": "the_execution_url_inside_ui",
|
||||
}
|
||||
```
|
||||
If you set the `wait` property to `true` and `returnOutputs` to `true`, the webhook call will wait for the flow to finish and return the flow outputs as response.
|
||||
|
||||
A webhook trigger can have conditions, but it doesn't support conditions of type `MultipleCondition`."""
|
||||
)
|
||||
@Plugin(
|
||||
@@ -148,23 +116,8 @@ public class Webhook extends AbstractTrigger implements TriggerOutput<Webhook.Ou
|
||||
|
||||
@PluginProperty
|
||||
@Builder.Default
|
||||
@Schema(
|
||||
title = "Wait for the flow to finish.",
|
||||
description = """
|
||||
If set to `true` the webhook call will wait for the flow to finish and return the flow outputs as response.
|
||||
If set to `false` the webhook call will return immediately after the execution is created.
|
||||
"""
|
||||
)
|
||||
private Boolean wait = false;
|
||||
|
||||
@PluginProperty
|
||||
@Builder.Default
|
||||
@Schema(
|
||||
title = "Send outputs of the flows as response for webhook caller.",
|
||||
description = "Requires `wait` to be `true`."
|
||||
)
|
||||
private Boolean returnOutputs = false;
|
||||
|
||||
public Optional<Execution> evaluate(HttpRequest<String> request, io.kestra.core.models.flows.Flow flow) {
|
||||
String body = request.getBody().orElse(null);
|
||||
|
||||
|
||||
@@ -21,13 +21,10 @@ import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@KestraTest
|
||||
@Execution(ExecutionMode.SAME_THREAD)
|
||||
class DocumentationGeneratorTest {
|
||||
@Inject
|
||||
JsonSchemaGenerator jsonSchemaGenerator;
|
||||
|
||||
@@ -37,7 +37,6 @@ import lombok.Value;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
@@ -68,7 +67,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@KestraTest
|
||||
@Testcontainers
|
||||
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
|
||||
class HttpClientTest {
|
||||
@Inject
|
||||
private ApplicationContext applicationContext;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user