mirror of
https://github.com/kestra-io/kestra.git
synced 2025-12-26 05:00:31 -05:00
Compare commits
187 Commits
debug-flak
...
v1.0.12
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe7a6d9af9 | ||
|
|
06c8c35061 | ||
|
|
8f23e813f2 | ||
|
|
47b7c7cd2e | ||
|
|
aca7c2f694 | ||
|
|
a0f29b7d5d | ||
|
|
0176c8c101 | ||
|
|
b0036bbfca | ||
|
|
fad5edbde8 | ||
|
|
f125f63ae5 | ||
|
|
6db1bfb2ce | ||
|
|
0957e07c78 | ||
|
|
5a4a5e44df | ||
|
|
faee3f1827 | ||
|
|
3604762da0 | ||
|
|
6ceb0de1d5 | ||
|
|
4a62f9c818 | ||
|
|
d14f3e3317 | ||
|
|
7e9030dfcf | ||
|
|
2fce17a8a9 | ||
|
|
67d8509106 | ||
|
|
01e92a6d79 | ||
|
|
883b7c8610 | ||
|
|
11ef823567 | ||
|
|
771cca1441 | ||
|
|
53e8674dfc | ||
|
|
59016ae1af | ||
|
|
7503d6fa21 | ||
|
|
0234a4c64c | ||
|
|
98c9c4d21f | ||
|
|
8e54183a44 | ||
|
|
8aa332c629 | ||
|
|
d10893ca00 | ||
|
|
c5ef356a1c | ||
|
|
0313e8e49b | ||
|
|
f4b6161f14 | ||
|
|
e69e82a35e | ||
|
|
e77378bcb7 | ||
|
|
3c9df90a35 | ||
|
|
6c86f0917c | ||
|
|
30b7346ee0 | ||
|
|
2f485c74ff | ||
|
|
3a5713bbd1 | ||
|
|
2eed738b83 | ||
|
|
5e2609ce5e | ||
|
|
86f909ce93 | ||
|
|
a8cb28a127 | ||
|
|
0fe9ba3e13 | ||
|
|
40f5aadd1a | ||
|
|
ceac25429a | ||
|
|
4144d9fbb1 | ||
|
|
9cc7d45f74 | ||
|
|
81ee330b9e | ||
|
|
5382655a2e | ||
|
|
483f7dc3b2 | ||
|
|
3c2da63837 | ||
|
|
31527891b2 | ||
|
|
6364f419d9 | ||
|
|
3c14432412 | ||
|
|
eaea4f5012 | ||
|
|
d43390a579 | ||
|
|
2404c36d35 | ||
|
|
bdbd217171 | ||
|
|
019c16af3c | ||
|
|
ff7d7c6a0b | ||
|
|
1042be87da | ||
|
|
104805d780 | ||
|
|
33c8e54f36 | ||
|
|
ff2e00d1ca | ||
|
|
0fe3f317c7 | ||
|
|
f753d15c91 | ||
|
|
c03e31de68 | ||
|
|
9a79f9a64c | ||
|
|
41468652d4 | ||
|
|
bc182277de | ||
|
|
8c2271089c | ||
|
|
9973a2120b | ||
|
|
bdfd038d40 | ||
|
|
a3fd734082 | ||
|
|
553a1d5389 | ||
|
|
c58aca967b | ||
|
|
27dcf60770 | ||
|
|
4e7c75232a | ||
|
|
f452da7ce1 | ||
|
|
43401c5017 | ||
|
|
067b110cf0 | ||
|
|
4ceff83a28 | ||
|
|
5026afe5bf | ||
|
|
3c899fcb2f | ||
|
|
cee412ffa9 | ||
|
|
3a57a683be | ||
|
|
a0b9de934e | ||
|
|
d677317cc5 | ||
|
|
9e661195e5 | ||
|
|
09c921bee5 | ||
|
|
d21ec4e899 | ||
|
|
efdb25fa97 | ||
|
|
37bdcc342c | ||
|
|
6d35f2b7a6 | ||
|
|
fe46ddf381 | ||
|
|
359dc9adc0 | ||
|
|
39c930124f | ||
|
|
1686fc3b4e | ||
|
|
03ff25ff55 | ||
|
|
d02fd53287 | ||
|
|
6c16bbe853 | ||
|
|
aa7a473d49 | ||
|
|
95133ebc40 | ||
|
|
54482e1d06 | ||
|
|
54b7811812 | ||
|
|
050ad60a95 | ||
|
|
030627ba7b | ||
|
|
c06ef7958f | ||
|
|
692d046289 | ||
|
|
92c1f04ec0 | ||
|
|
9e11d5fe5e | ||
|
|
14952c9457 | ||
|
|
ae314c301d | ||
|
|
f8aa5fb6ba | ||
|
|
c87d7e4da0 | ||
|
|
c928f1d822 | ||
|
|
baa07dd02b | ||
|
|
260cb50651 | ||
|
|
0a45325c69 | ||
|
|
c2522e2544 | ||
|
|
27476279ae | ||
|
|
3cc6372cb5 | ||
|
|
5f6e9dbe06 | ||
|
|
5078ce741d | ||
|
|
b7e17b7114 | ||
|
|
acaee34b0e | ||
|
|
1d78332505 | ||
|
|
7249632510 | ||
|
|
4a66a08c3b | ||
|
|
22fd6e97ea | ||
|
|
9afd86d32b | ||
|
|
797ea6c9e4 | ||
|
|
07d5e815c4 | ||
|
|
33ac9b1495 | ||
|
|
4d5b95d040 | ||
|
|
667aca7345 | ||
|
|
e05cc65202 | ||
|
|
71b606c27c | ||
|
|
47f9f12ce8 | ||
|
|
01acae5e97 | ||
|
|
e5878f08b7 | ||
|
|
0bcb6b4e0d | ||
|
|
3c2ecf4342 | ||
|
|
3d4f66772e | ||
|
|
e2afd4bcc3 | ||
|
|
d143097f03 | ||
|
|
72c0d91c1a | ||
|
|
1d692e56b0 | ||
|
|
0352d617ac | ||
|
|
b41aa4e0b9 | ||
|
|
d811dc030b | ||
|
|
105e62eee1 | ||
|
|
28796862a4 | ||
|
|
637cd794a4 | ||
|
|
fdd5c6e63d | ||
|
|
eda2483ec9 | ||
|
|
7b3c296489 | ||
|
|
fe6f8b4ed9 | ||
|
|
17ff539690 | ||
|
|
bbd0dda47e | ||
|
|
27a8e8b5a7 | ||
|
|
d6620a34cd | ||
|
|
6f8b3c5cfd | ||
|
|
6da6cbab60 | ||
|
|
a899e16178 | ||
|
|
568cd0b0c7 | ||
|
|
92e1dcb6eb | ||
|
|
499e040cd0 | ||
|
|
5916831d62 | ||
|
|
0b1b55957e | ||
|
|
7ee40d376a | ||
|
|
e2c9b3e256 | ||
|
|
556730777b | ||
|
|
c1a75a431f | ||
|
|
4a5b91667a | ||
|
|
f7b2af16a1 | ||
|
|
9351cb22e0 | ||
|
|
b1ecb82fdc | ||
|
|
c6d56151eb | ||
|
|
ed4398467a | ||
|
|
c51947419a | ||
|
|
ccb6a1f4a7 |
@@ -32,6 +32,11 @@ In the meantime, you can move onto the next step...
|
||||
|
||||
### Development:
|
||||
|
||||
- (Optional) By default, your dev server will target `localhost:8080`. If your backend is running elsewhere, you can create `.env.development.local` under `ui` folder with this content:
|
||||
```
|
||||
VITE_APP_API_URL={myApiUrl}
|
||||
```
|
||||
|
||||
- Navigate into the `ui` folder and run `npm install` to install the dependencies for the frontend project.
|
||||
|
||||
- Now go to the `cli/src/main/resources` folder and create a `application-override.yml` file.
|
||||
|
||||
4
.github/CONTRIBUTING.md
vendored
4
.github/CONTRIBUTING.md
vendored
@@ -32,7 +32,7 @@ Watch out for duplicates! If you are creating a new issue, please check existing
|
||||
#### Requirements
|
||||
The following dependencies are required to build Kestra locally:
|
||||
- Java 21+
|
||||
- Node 22+ and npm 10+
|
||||
- Node 18+ and npm
|
||||
- Python 3, pip and python venv
|
||||
- Docker & Docker Compose
|
||||
- an IDE (Intellij IDEA, Eclipse or VS Code)
|
||||
@@ -126,7 +126,7 @@ By default, Kestra will be installed under: `$HOME/.kestra/current`. Set the `KE
|
||||
```bash
|
||||
# build and install Kestra
|
||||
make install
|
||||
# install plugins (plugins installation is based on the API).
|
||||
# install plugins (plugins installation is based on the `.plugins` or `.plugins.override` files located at the root of the project.
|
||||
make install-plugins
|
||||
# start Kestra in standalone mode with Postgres as backend
|
||||
make start-standalone-postgres
|
||||
|
||||
11
.github/ISSUE_TEMPLATE/bug.yml
vendored
11
.github/ISSUE_TEMPLATE/bug.yml
vendored
@@ -1,13 +1,10 @@
|
||||
name: Bug report
|
||||
description: Report a bug or unexpected behavior in the project
|
||||
|
||||
labels: ["bug", "area/backend", "area/frontend"]
|
||||
|
||||
description: File a bug report
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack). Don't forget to give us a star! ⭐
|
||||
Thanks for reporting an issue! Please provide a [Minimal Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) and share any additional information that may help reproduce, troubleshoot, and hopefully fix the issue, including screenshots, error traceback, and your Kestra server logs. For quick questions, you can contact us directly on [Slack](https://kestra.io/slack).
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Describe the issue
|
||||
@@ -23,3 +20,7 @@ body:
|
||||
- Kestra Version: develop
|
||||
validations:
|
||||
required: false
|
||||
labels:
|
||||
- bug
|
||||
- area/backend
|
||||
- area/frontend
|
||||
|
||||
2
.github/ISSUE_TEMPLATE/config.yml
vendored
2
.github/ISSUE_TEMPLATE/config.yml
vendored
@@ -1,4 +1,4 @@
|
||||
contact_links:
|
||||
- name: Chat
|
||||
url: https://kestra.io/slack
|
||||
about: Chat with us on Slack
|
||||
about: Chat with us on Slack.
|
||||
11
.github/ISSUE_TEMPLATE/feature.yml
vendored
11
.github/ISSUE_TEMPLATE/feature.yml
vendored
@@ -1,12 +1,13 @@
|
||||
name: Feature request
|
||||
description: Suggest a new feature or improvement to enhance the project
|
||||
|
||||
labels: ["enhancement", "area/backend", "area/frontend"]
|
||||
|
||||
description: Create a new feature request
|
||||
body:
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Feature description
|
||||
placeholder: Tell us more about your feature request. Don't forget to give us a star! ⭐
|
||||
placeholder: Tell us more about your feature request
|
||||
validations:
|
||||
required: true
|
||||
labels:
|
||||
- enhancement
|
||||
- area/backend
|
||||
- area/frontend
|
||||
|
||||
4
.github/dependabot.yml
vendored
4
.github/dependabot.yml
vendored
@@ -26,10 +26,6 @@ updates:
|
||||
open-pull-requests-limit: 50
|
||||
labels:
|
||||
- "dependency-upgrade"
|
||||
ignore:
|
||||
- dependency-name: "com.google.protobuf:*"
|
||||
# Ignore versions of Protobuf that are equal to or greater than 4.0.0 as Orc still uses 3
|
||||
versions: [ "[4,)" ]
|
||||
|
||||
# Maintain dependencies for NPM modules
|
||||
- package-ecosystem: "npm"
|
||||
|
||||
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@@ -35,4 +35,4 @@ Remove this section if this change applies to all flows or to the documentation
|
||||
|
||||
If there are no setup requirements, you can remove this section.
|
||||
|
||||
Thank you for your contribution. ❤️ Don't forget to give us a star! ⭐ -->
|
||||
Thank you for your contribution. ❤️ -->
|
||||
|
||||
67
.github/workflows/auto-translate-ui-keys.yml
vendored
67
.github/workflows/auto-translate-ui-keys.yml
vendored
@@ -1,67 +0,0 @@
|
||||
name: Auto-Translate UI keys and create PR
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9-21/3 * * 1-5" # Every 3 hours from 9 AM to 9 PM, Monday to Friday
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
retranslate_modified_keys:
|
||||
description: "Whether to re-translate modified keys even if they already have translations."
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
default: "false"
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
translations:
|
||||
name: Translations
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
name: Checkout
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: pip install gitpython openai
|
||||
|
||||
- name: Generate translations
|
||||
run: python ui/src/translations/generate_translations.py ${{ github.event.inputs.retranslate_modified_keys }}
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "20.x"
|
||||
|
||||
- name: Set up Git
|
||||
run: |
|
||||
git config --global user.name "GitHub Action"
|
||||
git config --global user.email "actions@github.com"
|
||||
|
||||
- name: Commit and create PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH_NAME="chore/update-translations-$(date +%s)"
|
||||
git checkout -b $BRANCH_NAME
|
||||
git add ui/src/translations/*.json
|
||||
if git diff --cached --quiet; then
|
||||
echo "No changes to commit. Exiting with success."
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "chore(core): localize to languages other than english" -m "Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference."
|
||||
git push -u origin $BRANCH_NAME || (git push origin --delete $BRANCH_NAME && git push -u origin $BRANCH_NAME)
|
||||
gh pr create --title "Translations from en.json" --body $'This PR was created automatically by a GitHub Action.\n\nSomeone from the @kestra-io/frontend team needs to review and merge.' --base ${{ github.ref_name }} --head $BRANCH_NAME
|
||||
|
||||
- name: Check keys matching
|
||||
run: node ui/src/translations/check.js
|
||||
85
.github/workflows/codeql-analysis.yml
vendored
85
.github/workflows/codeql-analysis.yml
vendored
@@ -1,85 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 5 * * 1'
|
||||
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# Override automatic language detection by changing the below list
|
||||
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
|
||||
language: ['java', 'javascript']
|
||||
# Learn more...
|
||||
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
# We must fetch at least the immediate parents so that if this is
|
||||
# a pull request then we can checkout the head.
|
||||
fetch-depth: 2
|
||||
|
||||
# If this run was triggered by a pull request event, then checkout
|
||||
# the head of the pull request instead of the merge commit.
|
||||
- run: git checkout HEAD^2
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Set up JDK
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: 21
|
||||
|
||||
- name: Setup gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
|
||||
- name: Build with Gradle
|
||||
if: ${{ matrix.language == 'java' }}
|
||||
run: ./gradlew testClasses -x :ui:assembleFrontend
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
if: ${{ matrix.language != 'java' }}
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
15
.github/workflows/e2e-scheduling.yml
vendored
15
.github/workflows/e2e-scheduling.yml
vendored
@@ -1,15 +0,0 @@
|
||||
name: 'E2E tests scheduling'
|
||||
# 'New E2E tests implementation started by Roman. Based on playwright in npm UI project, tests Kestra OSS develop docker image. These tests are written from zero, lets make them unflaky from the start!.'
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 * * * *" # Every hour
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
noInputYet:
|
||||
description: 'not input yet.'
|
||||
required: false
|
||||
type: string
|
||||
default: "no input"
|
||||
jobs:
|
||||
e2e:
|
||||
uses: kestra-io/actions/.github/workflows/kestra-oss-e2e-tests.yml@main
|
||||
@@ -1,85 +0,0 @@
|
||||
name: Create new release branch
|
||||
run-name: "Create new release branch Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.0)'
|
||||
required: true
|
||||
type: string
|
||||
nextVersion:
|
||||
description: 'The next version (e.g., 0.22.0-SNAPSHOT)'
|
||||
required: true
|
||||
type: string
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
NEXT_VERSION: "${{ github.event.inputs.nextVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
steps:
|
||||
# Checks
|
||||
- name: Check Inputs
|
||||
run: |
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)\.0$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$NEXT_VERSION" =~ ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$ ]]; then
|
||||
echo "Invalid next version. Must match regex: ^[0-9]+(\.[0-9]+)\.0-SNAPSHOT$"
|
||||
exit 1;
|
||||
fi
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: kestra
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: true
|
||||
node-enabled: true
|
||||
python-enabled: true
|
||||
caches-enabled: true
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Run Gradle Release
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
PUSH_RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
|
||||
cd kestra
|
||||
|
||||
# Create and push release branch
|
||||
git checkout -B "$PUSH_RELEASE_BRANCH";
|
||||
git pull origin "$PUSH_RELEASE_BRANCH" --rebase || echo "No existing branch to pull";
|
||||
git push -u origin "$PUSH_RELEASE_BRANCH";
|
||||
|
||||
# Run gradle release
|
||||
git checkout develop;
|
||||
|
||||
if [[ "$RELEASE_VERSION" == *"-SNAPSHOT" ]]; then
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}" \
|
||||
-Prelease.failOnSnapshotDependencies=false
|
||||
else
|
||||
./gradlew release -Prelease.useAutomaticVersion=true \
|
||||
-Prelease.releaseVersion="${RELEASE_VERSION}" \
|
||||
-Prelease.newVersion="${NEXT_VERSION}" \
|
||||
-Prelease.pushReleaseVersionBranch="${PUSH_RELEASE_BRANCH}"
|
||||
fi
|
||||
65
.github/workflows/global-start-release.yml
vendored
65
.github/workflows/global-start-release.yml
vendored
@@ -1,65 +0,0 @@
|
||||
name: Start release
|
||||
run-name: "Start release of Kestra ${{ github.event.inputs.releaseVersion }} 🚀"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
releaseVersion:
|
||||
description: 'The release version (e.g., 0.21.1)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
RELEASE_VERSION: "${{ github.event.inputs.releaseVersion }}"
|
||||
jobs:
|
||||
release:
|
||||
name: Release Kestra
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Parse and Check Inputs
|
||||
id: parse-and-check-inputs
|
||||
run: |
|
||||
CURRENT_BRANCH="${{ github.ref_name }}"
|
||||
if ! [[ "$CURRENT_BRANCH" == "develop" ]]; then
|
||||
echo "You can only run this workflow on develop, but you ran it on $CURRENT_BRANCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$RELEASE_VERSION" =~ ^[0-9]+(\.[0-9]+)(\.[0-9]+)(-rc[0-9])?(-SNAPSHOT)?$ ]]; then
|
||||
echo "Invalid release version. Must match regex: ^[0-9]+(\.[0-9]+)(\.[0-9]+)-(rc[0-9])?(-SNAPSHOT)?$"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract the major and minor versions
|
||||
BASE_VERSION=$(echo "$RELEASE_VERSION" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
|
||||
RELEASE_BRANCH="releases/v${BASE_VERSION}.x"
|
||||
echo "release_branch=${RELEASE_BRANCH}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Checkout
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
ref: ${{ steps.parse-and-check-inputs.outputs.release_branch }}
|
||||
|
||||
# Configure
|
||||
- name: Git - Configure
|
||||
run: |
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config --global user.name "github-actions[bot]"
|
||||
|
||||
# Execute
|
||||
- name: Start release by updating version and pushing a new tag
|
||||
env:
|
||||
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
run: |
|
||||
# Update version
|
||||
sed -i "s/^version=.*/version=$RELEASE_VERSION/" ./gradle.properties
|
||||
git add ./gradle.properties
|
||||
git commit -m"chore(version): update to version '$RELEASE_VERSION'"
|
||||
git push
|
||||
git tag -a "v$RELEASE_VERSION" -m"v$RELEASE_VERSION"
|
||||
git push --tags
|
||||
14
.github/workflows/main-build.yml
vendored
14
.github/workflows/main-build.yml
vendored
@@ -67,24 +67,20 @@ jobs:
|
||||
|
||||
end:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [backend-tests, frontend-tests, publish-develop-docker, publish-develop-maven]
|
||||
if: "always() && github.repository == 'kestra-io/kestra'"
|
||||
needs: [publish-develop-docker, publish-develop-maven]
|
||||
if: always()
|
||||
steps:
|
||||
- run: echo "end CI of failed or success"
|
||||
- name: Trigger EE Workflow
|
||||
uses: peter-evans/repository-dispatch@5fc4efd1a4797ddb68ffd0714a238564e4cc0e6f # v4
|
||||
if: "!contains(needs.*.result, 'failure') && github.ref == 'refs/heads/develop'"
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
if: github.ref == 'refs/heads/develop' && needs.release.result == 'success'
|
||||
with:
|
||||
token: ${{ secrets.GH_PERSONAL_TOKEN }}
|
||||
repository: kestra-io/kestra-ee
|
||||
event-type: "oss-updated"
|
||||
|
||||
# Slack
|
||||
- run: echo "mark job as failure to forward error to Slack action" && exit 1
|
||||
if: ${{ contains(needs.*.result, 'failure') }}
|
||||
- name: Slack - Notification
|
||||
if: ${{ always() && contains(needs.*.result, 'failure') }}
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') }}
|
||||
uses: kestra-io/actions/composite/slack-status@main
|
||||
with:
|
||||
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
channel: 'C09FF36GKE1'
|
||||
|
||||
76
.github/workflows/vulnerabilities-check.yml
vendored
76
.github/workflows/vulnerabilities-check.yml
vendored
@@ -43,82 +43,8 @@ jobs:
|
||||
|
||||
# Upload dependency check report
|
||||
- name: Upload dependency check report
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: dependency-check-report
|
||||
path: build/reports/dependency-check-report.html
|
||||
|
||||
develop-image-check:
|
||||
name: Image Check (develop)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:develop
|
||||
format: 'template'
|
||||
template: '@/contrib/sarif.tpl'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
skip-dirs: /app/plugins
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
|
||||
latest-image-check:
|
||||
name: Image Check (latest)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
actions: read
|
||||
steps:
|
||||
# Checkout
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Setup build
|
||||
- uses: kestra-io/actions/composite/setup-build@main
|
||||
id: build
|
||||
with:
|
||||
java-enabled: false
|
||||
node-enabled: false
|
||||
|
||||
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
|
||||
- name: Docker Vulnerabilities Check
|
||||
uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1
|
||||
with:
|
||||
image-ref: kestra/kestra:latest
|
||||
format: table
|
||||
skip-dirs: /app/plugins
|
||||
scanners: vuln
|
||||
severity: 'CRITICAL,HIGH'
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
category: docker-
|
||||
1
.plugins
1
.plugins
@@ -66,7 +66,6 @@
|
||||
#plugin-jdbc:io.kestra.plugin:plugin-jdbc-sybase:LATEST
|
||||
#plugin-jenkins:io.kestra.plugin:plugin-jenkins:LATEST
|
||||
#plugin-jira:io.kestra.plugin:plugin-jira:LATEST
|
||||
#plugin-jms:io.kestra.plugin:plugin-jms:LATEST
|
||||
#plugin-kafka:io.kestra.plugin:plugin-kafka:LATEST
|
||||
#plugin-kestra:io.kestra.plugin:plugin-kestra:LATEST
|
||||
#plugin-kubernetes:io.kestra.plugin:plugin-kubernetes:LATEST
|
||||
|
||||
63
Makefile
63
Makefile
@@ -13,7 +13,7 @@ SHELL := /bin/bash
|
||||
|
||||
KESTRA_BASEDIR := $(shell echo $${KESTRA_HOME:-$$HOME/.kestra/current})
|
||||
KESTRA_WORKER_THREAD := $(shell echo $${KESTRA_WORKER_THREAD:-4})
|
||||
VERSION := $(shell awk -F= '/^version=/ {gsub(/-SNAPSHOT/, "", $$2); gsub(/[[:space:]]/, "", $$2); print $$2}' gradle.properties)
|
||||
VERSION := $(shell ./gradlew properties -q | awk '/^version:/ {print $$2}')
|
||||
GIT_COMMIT := $(shell git rev-parse --short HEAD)
|
||||
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
|
||||
DATE := $(shell date --rfc-3339=seconds)
|
||||
@@ -48,43 +48,38 @@ build-exec:
|
||||
./gradlew -q executableJar --no-daemon --priority=normal
|
||||
|
||||
install: build-exec
|
||||
@echo "Installing Kestra in ${KESTRA_BASEDIR}" ; \
|
||||
KESTRA_BASEDIR="${KESTRA_BASEDIR}" ; \
|
||||
mkdir -p "$${KESTRA_BASEDIR}/bin" "$${KESTRA_BASEDIR}/plugins" "$${KESTRA_BASEDIR}/flows" "$${KESTRA_BASEDIR}/logs" ; \
|
||||
echo "Copying executable..." ; \
|
||||
EXECUTABLE_FILE=$$(ls build/executable/kestra-* 2>/dev/null | head -n1) ; \
|
||||
if [ -z "$${EXECUTABLE_FILE}" ]; then \
|
||||
echo "[ERROR] No Kestra executable found in build/executable"; \
|
||||
exit 1; \
|
||||
fi ; \
|
||||
cp "$${EXECUTABLE_FILE}" "$${KESTRA_BASEDIR}/bin/kestra" ; \
|
||||
chmod +x "$${KESTRA_BASEDIR}/bin/kestra" ; \
|
||||
VERSION_INSTALLED=$$("$${KESTRA_BASEDIR}/bin/kestra" --version 2>/dev/null || echo "unknown") ; \
|
||||
echo "Kestra installed successfully (version=$${VERSION_INSTALLED}) 🚀"
|
||||
echo "Installing Kestra: ${KESTRA_BASEDIR}"
|
||||
mkdir -p ${KESTRA_BASEDIR}/bin ${KESTRA_BASEDIR}/plugins ${KESTRA_BASEDIR}/flows ${KESTRA_BASEDIR}/logs
|
||||
cp build/executable/* ${KESTRA_BASEDIR}/bin/kestra && chmod +x ${KESTRA_BASEDIR}/bin
|
||||
VERSION_INSTALLED=$$(${KESTRA_BASEDIR}/bin/kestra --version); \
|
||||
echo "Kestra installed successfully (version=$$VERSION_INSTALLED) 🚀"
|
||||
|
||||
# Install plugins for Kestra from the API.
|
||||
# Install plugins for Kestra from (.plugins file).
|
||||
install-plugins:
|
||||
@echo "Installing plugins for Kestra version ${VERSION}" ; \
|
||||
if [ -z "${VERSION}" ]; then \
|
||||
echo "[ERROR] Kestra version could not be determined."; \
|
||||
if [[ ! -f ".plugins" && ! -f ".plugins.override" ]]; then \
|
||||
echo "[ERROR] file '$$(pwd)/.plugins' and '$$(pwd)/.plugins.override' not found."; \
|
||||
exit 1; \
|
||||
fi ; \
|
||||
PLUGINS_PATH="${KESTRA_BASEDIR}/plugins" ; \
|
||||
echo "Fetching plugin list from Kestra API for version ${VERSION}..." ; \
|
||||
RESPONSE=$$(curl -s "https://api.kestra.io/v1/plugins/artifacts/core-compatibility/${VERSION}/latest") ; \
|
||||
if [ -z "$${RESPONSE}" ]; then \
|
||||
echo "[ERROR] Failed to fetch plugin list from API."; \
|
||||
exit 1; \
|
||||
fi ; \
|
||||
echo "Parsing plugin list (excluding EE and secret plugins)..." ; \
|
||||
echo "$${RESPONSE}" | jq -r '.[] | select(.license == "OPEN_SOURCE" and (.groupId != "io.kestra.plugin.ee") and (.groupId != "io.kestra.ee.secret")) | .groupId + ":" + .artifactId + ":" + .version' | while read -r plugin; do \
|
||||
[[ $$plugin =~ ^#.* ]] && continue ; \
|
||||
CURRENT_PLUGIN=$${plugin} ; \
|
||||
echo "Installing $$CURRENT_PLUGIN..." ; \
|
||||
fi; \
|
||||
|
||||
PLUGIN_LIST="./.plugins"; \
|
||||
if [[ -f ".plugins.override" ]]; then \
|
||||
PLUGIN_LIST="./.plugins.override"; \
|
||||
fi; \
|
||||
while IFS= read -r plugin; do \
|
||||
[[ $$plugin =~ ^#.* ]] && continue; \
|
||||
PLUGINS_PATH="${KESTRA_INSTALL_DIR}/plugins"; \
|
||||
CURRENT_PLUGIN=$${plugin/LATEST/"${VERSION}"}; \
|
||||
CURRENT_PLUGIN=$$(echo $$CURRENT_PLUGIN | cut -d':' -f2-); \
|
||||
PLUGIN_FILE="$$PLUGINS_PATH/$$(echo $$CURRENT_PLUGIN | awk -F':' '{print $$2"-"$$3}').jar"; \
|
||||
echo "Installing Kestra plugin $$CURRENT_PLUGIN > ${KESTRA_INSTALL_DIR}/plugins"; \
|
||||
if [ -f "$$PLUGIN_FILE" ]; then \
|
||||
echo "Plugin already installed in > $$PLUGIN_FILE"; \
|
||||
else \
|
||||
${KESTRA_BASEDIR}/bin/kestra plugins install $$CURRENT_PLUGIN \
|
||||
--plugins ${KESTRA_BASEDIR}/plugins \
|
||||
--repositories=https://central.sonatype.com/repository/maven-snapshots || exit 1 ; \
|
||||
done
|
||||
--plugins ${KESTRA_BASEDIR}/plugins \
|
||||
--repositories=https://central.sonatype.com/repository/maven-snapshots || exit 1; \
|
||||
fi \
|
||||
done < $$PLUGIN_LIST
|
||||
|
||||
# Build docker image from Kestra source.
|
||||
build-docker: build-exec
|
||||
|
||||
21
README.md
21
README.md
@@ -19,12 +19,9 @@
|
||||
<br />
|
||||
|
||||
<p align="center">
|
||||
<a href="https://twitter.com/kestra_io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/twitter.svg" alt="twitter" width="35" height="25" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" width="35" height="25" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io" style="margin: 0 10px;">
|
||||
<img height="25" src="https://kestra.io/youtube.svg" alt="youtube" width="35" height="25" /></a>
|
||||
<a href="https://x.com/kestra_io"><img height="25" src="https://kestra.io/twitter.svg" alt="X(formerly Twitter)" /></a>
|
||||
<a href="https://www.linkedin.com/company/kestra/"><img height="25" src="https://kestra.io/linkedin.svg" alt="linkedin" /></a>
|
||||
<a href="https://www.youtube.com/@kestra-io"><img height="25" src="https://kestra.io/youtube.svg" alt="youtube" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -36,10 +33,10 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.kestra.io/video/product-overview" target="_blank">
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
|
||||
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
|
||||
</a>
|
||||
</p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
|
||||
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
|
||||
|
||||
|
||||
## 🌟 What is Kestra?
|
||||
@@ -68,12 +65,6 @@ Kestra is an open-source, event-driven orchestration platform that makes both **
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Launch on AWS (CloudFormation)
|
||||
|
||||
Deploy Kestra on AWS using our CloudFormation template:
|
||||
|
||||
[](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?templateURL=https://kestra-deployment-templates.s3.eu-west-3.amazonaws.com/aws/cloudformation/ec2-rds-s3/kestra-oss.yaml&stackName=kestra-oss)
|
||||
|
||||
### Get Started Locally in 5 Minutes
|
||||
|
||||
#### Launch Kestra in Docker
|
||||
@@ -104,7 +95,7 @@ If you're on Windows and use WSL (Linux-based environment in Windows):
|
||||
```bash
|
||||
docker run --pull=always --rm -it -p 8080:8080 --user=root \
|
||||
-v "/var/run/docker.sock:/var/run/docker.sock" \
|
||||
-v "/mnt/c/Temp:/tmp" kestra/kestra:latest server local
|
||||
-v "C:/Temp:/tmp" kestra/kestra:latest server local
|
||||
```
|
||||
|
||||
Check our [Installation Guide](https://kestra.io/docs/installation) for other deployment options (Docker Compose, Podman, Kubernetes, AWS, GCP, Azure, and more).
|
||||
|
||||
25
build.gradle
25
build.gradle
@@ -21,23 +21,23 @@ plugins {
|
||||
|
||||
// test
|
||||
id "com.adarshr.test-logger" version "4.0.0"
|
||||
id "org.sonarqube" version "7.0.1.6134"
|
||||
id "org.sonarqube" version "6.3.1.5724"
|
||||
id 'jacoco-report-aggregation'
|
||||
|
||||
// helper
|
||||
id "com.github.ben-manes.versions" version "0.53.0"
|
||||
id "com.github.ben-manes.versions" version "0.52.0"
|
||||
|
||||
// front
|
||||
id 'com.github.node-gradle.node' version '7.1.0'
|
||||
|
||||
// release
|
||||
id 'net.researchgate.release' version '3.1.0'
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.3"
|
||||
id "com.gorylenko.gradle-git-properties" version "2.5.2"
|
||||
id 'signing'
|
||||
id "com.vanniktech.maven.publish" version "0.34.0"
|
||||
|
||||
// OWASP dependency check
|
||||
id "org.owasp.dependencycheck" version "12.1.8" apply false
|
||||
id "org.owasp.dependencycheck" version "12.1.3" apply false
|
||||
}
|
||||
|
||||
idea {
|
||||
@@ -168,9 +168,8 @@ allprojects {
|
||||
/**********************************************************************************************************************\
|
||||
* Test
|
||||
**********************************************************************************************************************/
|
||||
subprojects {subProj ->
|
||||
|
||||
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
|
||||
subprojects {
|
||||
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
|
||||
apply plugin: "com.adarshr.test-logger"
|
||||
|
||||
java {
|
||||
@@ -222,14 +221,6 @@ subprojects {subProj ->
|
||||
t.environment 'ENV_TEST1', "true"
|
||||
t.environment 'ENV_TEST2', "Pass by env"
|
||||
|
||||
|
||||
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
|
||||
// JUnit 5 parallel settings
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
|
||||
t.systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('flakyTest', Test) { Test t ->
|
||||
@@ -331,7 +322,7 @@ subprojects {
|
||||
}
|
||||
|
||||
dependencies {
|
||||
agent "org.aspectj:aspectjweaver:1.9.25"
|
||||
agent "org.aspectj:aspectjweaver:1.9.24"
|
||||
}
|
||||
|
||||
test {
|
||||
@@ -372,7 +363,7 @@ tasks.named('testCodeCoverageReport') {
|
||||
subprojects {
|
||||
sonar {
|
||||
properties {
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml,$projectDir.parentFile.path/build/reports/jacoco/test/testCodeCoverageReport.xml"
|
||||
property "sonar.coverage.jacoco.xmlReportPaths", "$projectDir.parentFile.path/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,5 @@ dependencies {
|
||||
implementation project(":worker")
|
||||
|
||||
//test
|
||||
testImplementation project(':tests')
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
@@ -43,13 +43,13 @@ import java.util.concurrent.Callable;
|
||||
SysCommand.class,
|
||||
ConfigCommand.class,
|
||||
NamespaceCommand.class,
|
||||
MigrationCommand.class
|
||||
MigrationCommand.class,
|
||||
}
|
||||
)
|
||||
@Introspected
|
||||
public class App implements Callable<Integer> {
|
||||
public static void main(String[] args) {
|
||||
execute(App.class, new String [] { Environment.CLI }, args);
|
||||
execute(App.class, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
|
||||
return PicocliRunner.call(App.class, "--help");
|
||||
}
|
||||
|
||||
protected static void execute(Class<?> cls, String[] environments, String... args) {
|
||||
protected static void execute(Class<?> cls, String... args) {
|
||||
// Log Bridge
|
||||
SLF4JBridgeHandler.removeHandlersForRootLogger();
|
||||
SLF4JBridgeHandler.install();
|
||||
|
||||
// Init ApplicationContext
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
|
||||
ApplicationContext applicationContext = App.applicationContext(cls, args);
|
||||
|
||||
// Call Picocli command
|
||||
int exitCode = 0;
|
||||
@@ -80,7 +80,6 @@ public class App implements Callable<Integer> {
|
||||
System.exit(Objects.requireNonNullElse(exitCode, 0));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
|
||||
* forced Properties from current command.
|
||||
@@ -89,13 +88,12 @@ public class App implements Callable<Integer> {
|
||||
* @return the application context created
|
||||
*/
|
||||
protected static ApplicationContext applicationContext(Class<?> mainClass,
|
||||
String[] environments,
|
||||
String[] args) {
|
||||
|
||||
ApplicationContextBuilder builder = ApplicationContext
|
||||
.builder()
|
||||
.mainClass(mainClass)
|
||||
.environments(environments);
|
||||
.environments(Environment.CLI);
|
||||
|
||||
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
|
||||
continueOnParsingErrors(cmd);
|
||||
|
||||
@@ -2,7 +2,6 @@ package io.kestra.cli.commands.migrations;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.cli.App;
|
||||
import io.kestra.cli.commands.migrations.metadata.MetadataMigrationCommand;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -14,7 +13,6 @@ import picocli.CommandLine;
|
||||
mixinStandardHelpOptions = true,
|
||||
subcommands = {
|
||||
TenantMigrationCommand.class,
|
||||
MetadataMigrationCommand.class
|
||||
}
|
||||
)
|
||||
@Slf4j
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Provider;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@CommandLine.Command(
|
||||
name = "kv",
|
||||
description = "populate metadata for KV"
|
||||
)
|
||||
@Slf4j
|
||||
public class KvMetadataMigrationCommand extends AbstractCommand {
|
||||
@Inject
|
||||
private Provider<MetadataMigrationService> metadataMigrationServiceProvider;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
super.call();
|
||||
try {
|
||||
metadataMigrationServiceProvider.get().kvMigration();
|
||||
} catch (Exception e) {
|
||||
System.err.println("❌ KV Metadata migration failed: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
return 1;
|
||||
}
|
||||
System.out.println("✅ KV Metadata migration complete.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import jakarta.inject.Inject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@CommandLine.Command(
|
||||
name = "metadata",
|
||||
description = "populate metadata for entities",
|
||||
subcommands = {
|
||||
KvMetadataMigrationCommand.class,
|
||||
SecretsMetadataMigrationCommand.class
|
||||
}
|
||||
)
|
||||
@Slf4j
|
||||
public class MetadataMigrationCommand extends AbstractCommand {
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
super.call();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.KvMetadataRepositoryInterface;
|
||||
import io.kestra.core.storages.FileAttributes;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.storages.kv.InternalKVStore;
|
||||
import io.kestra.core.storages.kv.KVEntry;
|
||||
import io.kestra.core.tenant.TenantService;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
|
||||
@Singleton
|
||||
public class MetadataMigrationService {
|
||||
@Inject
|
||||
private TenantService tenantService;
|
||||
|
||||
@Inject
|
||||
private FlowRepositoryInterface flowRepository;
|
||||
|
||||
@Inject
|
||||
private KvMetadataRepositoryInterface kvMetadataRepository;
|
||||
|
||||
@Inject
|
||||
private StorageInterface storageInterface;
|
||||
|
||||
protected Map<String, List<String>> namespacesPerTenant() {
|
||||
String tenantId = tenantService.resolveTenant();
|
||||
return Map.of(tenantId, flowRepository.findDistinctNamespace(tenantId));
|
||||
}
|
||||
|
||||
public void kvMigration() throws IOException {
|
||||
this.namespacesPerTenant().entrySet().stream()
|
||||
.flatMap(namespacesForTenant -> namespacesForTenant.getValue().stream().map(namespace -> Map.entry(namespacesForTenant.getKey(), namespace)))
|
||||
.flatMap(throwFunction(namespaceForTenant -> {
|
||||
InternalKVStore kvStore = new InternalKVStore(namespaceForTenant.getKey(), namespaceForTenant.getValue(), storageInterface, kvMetadataRepository);
|
||||
List<FileAttributes> list = listAllFromStorage(storageInterface, namespaceForTenant.getKey(), namespaceForTenant.getValue());
|
||||
Map<Boolean, List<KVEntry>> entriesByIsExpired = list.stream()
|
||||
.map(throwFunction(fileAttributes -> KVEntry.from(namespaceForTenant.getValue(), fileAttributes)))
|
||||
.collect(Collectors.partitioningBy(kvEntry -> Optional.ofNullable(kvEntry.expirationDate()).map(expirationDate -> Instant.now().isAfter(expirationDate)).orElse(false)));
|
||||
|
||||
entriesByIsExpired.get(true).forEach(kvEntry -> {
|
||||
try {
|
||||
storageInterface.delete(
|
||||
namespaceForTenant.getKey(),
|
||||
namespaceForTenant.getValue(),
|
||||
kvStore.storageUri(kvEntry.key())
|
||||
);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
return entriesByIsExpired.get(false).stream().map(kvEntry -> PersistedKvMetadata.from(namespaceForTenant.getKey(), kvEntry));
|
||||
}))
|
||||
.forEach(throwConsumer(kvMetadata -> {
|
||||
if (kvMetadataRepository.findByName(kvMetadata.getTenantId(), kvMetadata.getNamespace(), kvMetadata.getName()).isEmpty()) {
|
||||
kvMetadataRepository.save(kvMetadata);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
public void secretMigration() throws Exception {
|
||||
throw new UnsupportedOperationException("Secret migration is not needed in the OSS version");
|
||||
}
|
||||
|
||||
private static List<FileAttributes> listAllFromStorage(StorageInterface storage, String tenant, String namespace) throws IOException {
|
||||
try {
|
||||
return storage.list(tenant, namespace, URI.create(StorageContext.KESTRA_PROTOCOL + StorageContext.kvPrefix(namespace)));
|
||||
} catch (FileNotFoundException e) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Provider;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@CommandLine.Command(
|
||||
name = "secrets",
|
||||
description = "populate metadata for secrets"
|
||||
)
|
||||
@Slf4j
|
||||
public class SecretsMetadataMigrationCommand extends AbstractCommand {
|
||||
@Inject
|
||||
private Provider<MetadataMigrationService> metadataMigrationServiceProvider;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
super.call();
|
||||
try {
|
||||
metadataMigrationServiceProvider.get().secretMigration();
|
||||
} catch (Exception e) {
|
||||
System.err.println("❌ Secrets Metadata migration failed: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
return 1;
|
||||
}
|
||||
System.out.println("✅ Secrets Metadata migration complete.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -2,27 +2,19 @@ package io.kestra.cli.commands.servers;
|
||||
|
||||
import io.kestra.cli.AbstractCommand;
|
||||
import io.kestra.core.contexts.KestraContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import picocli.CommandLine;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
|
||||
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
|
||||
Integer serverPort;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
|
||||
|
||||
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
|
||||
|
||||
return super.call();
|
||||
}
|
||||
|
||||
private long maxMemoryInMB() {
|
||||
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
|
||||
}
|
||||
|
||||
protected static int defaultWorkerThread() {
|
||||
return Runtime.getRuntime().availableProcessors() * 8;
|
||||
}
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
package io.kestra.cli.listeners;
|
||||
|
||||
import io.kestra.core.server.LocalServiceState;
|
||||
import io.kestra.core.server.Service;
|
||||
import io.kestra.core.server.ServiceRegistry;
|
||||
import io.micronaut.context.annotation.Requires;
|
||||
import io.micronaut.context.event.ApplicationEventListener;
|
||||
import io.micronaut.context.event.ShutdownEvent;
|
||||
import io.micronaut.core.annotation.Order;
|
||||
import io.micronaut.core.order.Ordered;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ForkJoinPool;
|
||||
|
||||
/**
|
||||
* Global application shutdown handler.
|
||||
* This handler gets effectively invoked before {@link jakarta.annotation.PreDestroy} does.
|
||||
*/
|
||||
@Singleton
|
||||
@Slf4j
|
||||
@Order(Ordered.LOWEST_PRECEDENCE)
|
||||
@Requires(property = "kestra.server-type")
|
||||
public class GracefulEmbeddedServiceShutdownListener implements ApplicationEventListener<ShutdownEvent> {
|
||||
@Inject
|
||||
ServiceRegistry serviceRegistry;
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
**/
|
||||
@Override
|
||||
public boolean supports(ShutdownEvent event) {
|
||||
return ApplicationEventListener.super.supports(event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for services' close actions
|
||||
*
|
||||
* @param event the event to respond to
|
||||
*/
|
||||
@Override
|
||||
public void onApplicationEvent(ShutdownEvent event) {
|
||||
List<LocalServiceState> states = serviceRegistry.all();
|
||||
if (states.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.debug("Shutdown event received");
|
||||
|
||||
List<CompletableFuture<Void>> futures = states.stream()
|
||||
.map(state -> CompletableFuture.runAsync(() -> closeService(state), ForkJoinPool.commonPool()))
|
||||
.toList();
|
||||
|
||||
// Wait for all services to close, before shutting down the embedded server
|
||||
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
|
||||
}
|
||||
|
||||
private void closeService(LocalServiceState state) {
|
||||
final Service service = state.service();
|
||||
try {
|
||||
service.unwrap().close();
|
||||
} catch (Exception e) {
|
||||
log.error("[Service id={}, type={}] Unexpected error on close", service.getId(), service.getType(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -262,8 +262,6 @@ public class FileChangedEventListener {
|
||||
}
|
||||
|
||||
private String getTenantIdFromPath(Path path) {
|
||||
// FIXME there is probably a bug here when a tenant has '_' in its name,
|
||||
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
|
||||
return path.getFileName().toString().split("_")[0];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,15 +30,15 @@ micronaut:
|
||||
read-idle-timeout: 60m
|
||||
write-idle-timeout: 60m
|
||||
idle-timeout: 60m
|
||||
netty:
|
||||
max-zstd-encode-size: 67108864 # increased to 64MB from the default of 32MB
|
||||
max-chunk-size: 10MB
|
||||
max-header-size: 32768 # increased from the default of 8k
|
||||
responses:
|
||||
file:
|
||||
cache-seconds: 86400
|
||||
cache-control:
|
||||
public: true
|
||||
netty:
|
||||
max-zstd-encode-size: 67108864 # increased to 64MB from the default of 32MB
|
||||
max-chunk-size: 10MB
|
||||
max-header-size: 32768 # increased from the default of 8k
|
||||
|
||||
# Access log configuration, see https://docs.micronaut.io/latest/guide/index.html#accessLogger
|
||||
access-logger:
|
||||
@@ -49,8 +49,6 @@ micronaut:
|
||||
- /ui/.+
|
||||
- /health
|
||||
- /health/.+
|
||||
- /metrics
|
||||
- /metrics/.+
|
||||
- /prometheus
|
||||
http-version: HTTP_1_1
|
||||
caches:
|
||||
@@ -243,10 +241,6 @@ kestra:
|
||||
ui-anonymous-usage-report:
|
||||
enabled: true
|
||||
|
||||
ui:
|
||||
charts:
|
||||
default-duration: P30D
|
||||
|
||||
anonymous-usage-report:
|
||||
enabled: true
|
||||
uri: https://api.kestra.io/v1/reports/server-events
|
||||
|
||||
@@ -37,7 +37,7 @@ class AppTest {
|
||||
|
||||
final String[] args = new String[]{"server", serverType, "--help"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
|
||||
|
||||
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
|
||||
@@ -52,7 +52,7 @@ class AppTest {
|
||||
|
||||
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
|
||||
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
|
||||
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
|
||||
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
|
||||
|
||||
assertThat(out.toString()).startsWith("Missing required parameters: ");
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package io.kestra.cli.commands.configs.sys;
|
||||
import io.kestra.cli.commands.flows.FlowCreateCommand;
|
||||
import io.kestra.cli.commands.namespaces.kv.KvCommand;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.runtime.server.EmbeddedServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
/**
|
||||
* Verifies CLI behavior without repository configuration:
|
||||
* - Repo-independent commands succeed (e.g. KV with no params).
|
||||
* - Repo-dependent commands fail with a clear error.
|
||||
*/
|
||||
class NoConfigCommandTest {
|
||||
|
||||
@Test
|
||||
void shouldSucceedWithNamespaceKVCommandWithoutParamsAndConfig() {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
System.setOut(new PrintStream(out));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder().deduceEnvironment(false).start()) {
|
||||
String[] args = {};
|
||||
Integer call = PicocliRunner.call(KvCommand.class, ctx, args);
|
||||
|
||||
assertThat(call).isZero();
|
||||
assertThat(out.toString()).contains("Usage: kestra namespace kv");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailWithCreateFlowCommandWithoutConfig() throws URISyntaxException {
|
||||
URL flowUrl = NoConfigCommandTest.class.getClassLoader().getResource("crudFlow/date.yml");
|
||||
Objects.requireNonNull(flowUrl, "Test flow resource not found");
|
||||
|
||||
Path flowPath = Paths.get(flowUrl.toURI());
|
||||
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
ByteArrayOutputStream err=new ByteArrayOutputStream();
|
||||
|
||||
System.setOut(new PrintStream(out));
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.builder()
|
||||
.deduceEnvironment(false)
|
||||
.start()) {
|
||||
|
||||
EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
|
||||
embeddedServer.start();
|
||||
|
||||
String[] createArgs = {
|
||||
"--server",
|
||||
embeddedServer.getURL().toString(),
|
||||
"--user",
|
||||
"myuser:pass:word",
|
||||
flowPath.toString(),
|
||||
};
|
||||
|
||||
Integer exitCode = PicocliRunner.call(FlowCreateCommand.class, ctx, createArgs);
|
||||
|
||||
|
||||
assertThat(exitCode).isNotZero();
|
||||
assertThat(out.toString()).isEmpty();
|
||||
assertThat(err.toString()).contains("No bean of type [io.kestra.core.repositories.FlowRepositoryInterface] exists");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.cli.App;
|
||||
import io.kestra.core.exceptions.ResourceExpiredException;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.repositories.KvMetadataRepositoryInterface;
|
||||
import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.storages.StorageObject;
|
||||
import io.kestra.core.storages.kv.*;
|
||||
import io.kestra.core.tenant.TenantService;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.kestra.plugin.core.log.Log;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.context.env.Environment;
|
||||
import io.micronaut.core.annotation.NonNull;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class KvMetadataMigrationCommandTest {
|
||||
@Test
|
||||
void run() throws IOException, ResourceExpiredException {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
System.setOut(new PrintStream(out));
|
||||
ByteArrayOutputStream err = new ByteArrayOutputStream();
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
|
||||
/* Initial setup:
|
||||
* - namespace 1: key, description, value
|
||||
* - namespace 1: expiredKey
|
||||
* - namespace 2: anotherKey, anotherDescription
|
||||
* - Nothing in database */
|
||||
String namespace = TestsUtils.randomNamespace();
|
||||
String key = "myKey";
|
||||
StorageInterface storage = ctx.getBean(StorageInterface.class);
|
||||
String description = "Some description";
|
||||
String value = "someValue";
|
||||
putOldKv(storage, namespace, key, description, value);
|
||||
|
||||
String anotherNamespace = TestsUtils.randomNamespace();
|
||||
String anotherKey = "anotherKey";
|
||||
String anotherDescription = "another description";
|
||||
putOldKv(storage, anotherNamespace, anotherKey, anotherDescription, "anotherValue");
|
||||
|
||||
String tenantId = TenantService.MAIN_TENANT;
|
||||
|
||||
// Expired KV should not be migrated + should be purged from the storage
|
||||
String expiredKey = "expiredKey";
|
||||
putOldKv(storage, namespace, expiredKey, Instant.now().minus(Duration.ofMinutes(5)), "some expired description", "expiredValue");
|
||||
assertThat(storage.exists(tenantId, null, getKvStorageUri(namespace, expiredKey))).isTrue();
|
||||
|
||||
KvMetadataRepositoryInterface kvMetadataRepository = ctx.getBean(KvMetadataRepositoryInterface.class);
|
||||
assertThat(kvMetadataRepository.findByName(tenantId, namespace, key).isPresent()).isFalse();
|
||||
|
||||
/* Expected outcome from the migration command:
|
||||
* - no KV has been migrated because no flow exist in the namespace so they are not picked up because we don't know they exist */
|
||||
String[] kvMetadataMigrationCommand = {
|
||||
"migrate", "metadata", "kv"
|
||||
};
|
||||
PicocliRunner.call(App.class, ctx, kvMetadataMigrationCommand);
|
||||
|
||||
|
||||
assertThat(out.toString()).contains("✅ KV Metadata migration complete.");
|
||||
// Still it's not in the metadata repository because no flow exist to find that kv
|
||||
assertThat(kvMetadataRepository.findByName(tenantId, namespace, key).isPresent()).isFalse();
|
||||
assertThat(kvMetadataRepository.findByName(tenantId, anotherNamespace, anotherKey).isPresent()).isFalse();
|
||||
|
||||
// A flow is created from namespace 1, so the KV in this namespace should be migrated
|
||||
FlowRepositoryInterface flowRepository = ctx.getBean(FlowRepositoryInterface.class);
|
||||
flowRepository.create(GenericFlow.of(Flow.builder()
|
||||
.tenantId(tenantId)
|
||||
.id("a-flow")
|
||||
.namespace(namespace)
|
||||
.tasks(List.of(Log.builder().id("log").type(Log.class.getName()).message("logging").build()))
|
||||
.build()));
|
||||
|
||||
/* We run the migration again:
|
||||
* - namespace 1 KV is seen and metadata is migrated to database
|
||||
* - namespace 2 KV is not seen because no flow exist in this namespace
|
||||
* - expiredKey is deleted from storage and not migrated */
|
||||
out.reset();
|
||||
PicocliRunner.call(App.class, ctx, kvMetadataMigrationCommand);
|
||||
|
||||
assertThat(out.toString()).contains("✅ KV Metadata migration complete.");
|
||||
Optional<PersistedKvMetadata> foundKv = kvMetadataRepository.findByName(tenantId, namespace, key);
|
||||
assertThat(foundKv.isPresent()).isTrue();
|
||||
assertThat(foundKv.get().getDescription()).isEqualTo(description);
|
||||
|
||||
assertThat(kvMetadataRepository.findByName(tenantId, anotherNamespace, anotherKey).isPresent()).isFalse();
|
||||
|
||||
KVStore kvStore = new InternalKVStore(tenantId, namespace, storage, kvMetadataRepository);
|
||||
Optional<KVEntry> actualKv = kvStore.get(key);
|
||||
assertThat(actualKv.isPresent()).isTrue();
|
||||
assertThat(actualKv.get().description()).isEqualTo(description);
|
||||
|
||||
Optional<KVValue> actualValue = kvStore.getValue(key);
|
||||
assertThat(actualValue.isPresent()).isTrue();
|
||||
assertThat(actualValue.get().value()).isEqualTo(value);
|
||||
|
||||
assertThat(kvMetadataRepository.findByName(tenantId, namespace, expiredKey).isPresent()).isFalse();
|
||||
assertThat(storage.exists(tenantId, null, getKvStorageUri(namespace, expiredKey))).isFalse();
|
||||
|
||||
/* We run one last time the migration without any change to verify that we don't resave an existing metadata.
|
||||
* It covers the case where user didn't perform the migrate command yet but they played and added some KV from the UI (so those ones will already be in metadata database). */
|
||||
out.reset();
|
||||
PicocliRunner.call(App.class, ctx, kvMetadataMigrationCommand);
|
||||
|
||||
assertThat(out.toString()).contains("✅ KV Metadata migration complete.");
|
||||
foundKv = kvMetadataRepository.findByName(tenantId, namespace, key);
|
||||
assertThat(foundKv.get().getVersion()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void putOldKv(StorageInterface storage, String namespace, String key, String description, String value) throws IOException {
|
||||
putOldKv(storage, namespace, key, Instant.now().plus(Duration.ofMinutes(5)), description, value);
|
||||
}
|
||||
|
||||
private static void putOldKv(StorageInterface storage, String namespace, String key, Instant expirationDate, String description, String value) throws IOException {
|
||||
URI kvStorageUri = getKvStorageUri(namespace, key);
|
||||
KVValueAndMetadata kvValueAndMetadata = new KVValueAndMetadata(new KVMetadata(description, expirationDate), value);
|
||||
storage.put(TenantService.MAIN_TENANT, namespace, kvStorageUri, new StorageObject(
|
||||
kvValueAndMetadata.metadataAsMap(),
|
||||
new ByteArrayInputStream(JacksonMapper.ofIon().writeValueAsBytes(kvValueAndMetadata.value()))
|
||||
));
|
||||
}
|
||||
|
||||
private static @NonNull URI getKvStorageUri(String namespace, String key) {
|
||||
return URI.create(StorageContext.KESTRA_PROTOCOL + StorageContext.kvPrefix(namespace) + "/" + key + ".ion");
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package io.kestra.cli.commands.migrations.metadata;
|
||||
|
||||
import io.kestra.cli.App;
|
||||
import io.micronaut.configuration.picocli.PicocliRunner;
|
||||
import io.micronaut.context.ApplicationContext;
|
||||
import io.micronaut.context.env.Environment;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class SecretsMetadataMigrationCommandTest {
|
||||
@Test
|
||||
void run() {
|
||||
ByteArrayOutputStream err = new ByteArrayOutputStream();
|
||||
System.setErr(new PrintStream(err));
|
||||
|
||||
try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
|
||||
String[] secretMetadataMigrationCommand = {
|
||||
"migrate", "metadata", "secrets"
|
||||
};
|
||||
PicocliRunner.call(App.class, ctx, secretMetadataMigrationCommand);
|
||||
|
||||
assertThat(err.toString()).contains("❌ Secrets Metadata migration failed: Secret migration is not needed in the OSS version");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,14 @@
|
||||
package io.kestra.cli.services;
|
||||
|
||||
import io.kestra.core.junit.annotations.FlakyTest;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.GenericFlow;
|
||||
import io.kestra.core.repositories.FlowRepositoryInterface;
|
||||
import io.kestra.core.utils.Await;
|
||||
import io.kestra.core.utils.TestsUtils;
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
|
||||
import jakarta.inject.Inject;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@@ -19,8 +18,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junitpioneer.jupiter.RetryingTest;
|
||||
|
||||
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
|
||||
import static io.kestra.core.utils.Rethrow.throwRunnable;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -58,12 +57,10 @@ class FileChangedEventListenerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void test() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a basic flow
|
||||
String flow = """
|
||||
@@ -76,14 +73,14 @@ class FileChangedEventListenerTest {
|
||||
message: Hello World! 🚀
|
||||
""";
|
||||
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
|
||||
assertThat(myflow.getTasks()).hasSize(1);
|
||||
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
|
||||
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -91,18 +88,16 @@ class FileChangedEventListenerTest {
|
||||
// delete the flow
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
}
|
||||
|
||||
@FlakyTest
|
||||
@RetryingTest(2)
|
||||
@RetryingTest(5) // Flaky on CI but always pass locally
|
||||
void testWithPluginDefault() throws IOException, TimeoutException {
|
||||
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
|
||||
// remove the flow if it already exists
|
||||
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
|
||||
|
||||
// create a flow with plugin default
|
||||
String pluginDefault = """
|
||||
@@ -118,14 +113,14 @@ class FileChangedEventListenerTest {
|
||||
values:
|
||||
message: Hello World!
|
||||
""";
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
|
||||
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
|
||||
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
|
||||
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
|
||||
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
|
||||
@@ -133,7 +128,7 @@ class FileChangedEventListenerTest {
|
||||
// delete both files
|
||||
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
|
||||
Await.until(
|
||||
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
|
||||
Duration.ofMillis(100),
|
||||
Duration.ofSeconds(10)
|
||||
);
|
||||
|
||||
@@ -84,7 +84,7 @@ dependencies {
|
||||
|
||||
testImplementation "org.testcontainers:testcontainers:1.21.3"
|
||||
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on"
|
||||
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
|
||||
|
||||
testImplementation "org.wiremock:wiremock-jetty12"
|
||||
}
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
package io.kestra.core.exceptions;
|
||||
|
||||
public class InvalidTriggerConfigurationException extends KestraRuntimeException {
|
||||
public InvalidTriggerConfigurationException() {
|
||||
super();
|
||||
}
|
||||
|
||||
public InvalidTriggerConfigurationException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public InvalidTriggerConfigurationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -91,13 +91,11 @@ public class HttpConfiguration {
|
||||
@Deprecated
|
||||
private final String proxyPassword;
|
||||
|
||||
@Schema(title = "The username for HTTP basic authentication. " +
|
||||
"Deprecated, use `auth` property with a `BasicAuthConfiguration` instance instead.")
|
||||
@Schema(title = "The username for HTTP basic authentication.")
|
||||
@Deprecated
|
||||
private final String basicAuthUser;
|
||||
|
||||
@Schema(title = "The password for HTTP basic authentication. " +
|
||||
"Deprecated, use `auth` property with a `BasicAuthConfiguration` instance instead.")
|
||||
@Schema(title = "The password for HTTP basic authentication.")
|
||||
@Deprecated
|
||||
private final String basicAuthPassword;
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
public enum FetchVersion {
|
||||
LATEST,
|
||||
OLD,
|
||||
ALL
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Schema(description = "A key/value pair that can be attached to a Flow or Execution. Labels are often used to organize and categorize objects.")
|
||||
@@ -44,7 +43,7 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static Map<String, String> toMap(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
|
||||
return labels.stream()
|
||||
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
|
||||
.filter(label -> label.value() != null && label.key() != null)
|
||||
// using an accumulator in case labels with the same key exists: the second is kept
|
||||
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
|
||||
}
|
||||
@@ -59,7 +58,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
public static List<Label> deduplicate(@Nullable List<Label> labels) {
|
||||
if (labels == null || labels.isEmpty()) return Collections.emptyList();
|
||||
return toMap(labels).entrySet().stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toCollection(ArrayList::new));
|
||||
}
|
||||
@@ -74,7 +72,6 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
if (map == null || map.isEmpty()) return List.of();
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(getEntryNotEmptyPredicate())
|
||||
.map(entry -> new Label(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
@@ -93,14 +90,4 @@ public record Label(@NotEmpty String key, @NotEmpty String value) {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides predicate for not empty entries.
|
||||
*
|
||||
* @return The non-empty filter
|
||||
*/
|
||||
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
|
||||
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
|
||||
entry.getValue() != null && !entry.getValue().isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,16 +91,10 @@ public record QueryFilter(
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX, Op.IN, Op.NOT_IN, Op.PREFIX);
|
||||
}
|
||||
},
|
||||
KIND("kind") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.EQUALS,Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
LABELS("labels") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.IN, Op.NOT_IN, Op.CONTAINS);
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
FLOW_ID("flowId") {
|
||||
@@ -109,12 +103,6 @@ public record QueryFilter(
|
||||
return List.of(Op.EQUALS, Op.NOT_EQUALS, Op.CONTAINS, Op.STARTS_WITH, Op.ENDS_WITH, Op.REGEX);
|
||||
}
|
||||
},
|
||||
UPDATED("updated") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
return List.of(Op.GREATER_THAN_OR_EQUAL_TO, Op.GREATER_THAN, Op.LESS_THAN_OR_EQUAL_TO, Op.LESS_THAN, Op.EQUALS, Op.NOT_EQUALS);
|
||||
}
|
||||
},
|
||||
START_DATE("startDate") {
|
||||
@Override
|
||||
public List<Op> supportedOp() {
|
||||
@@ -223,7 +211,7 @@ public record QueryFilter(
|
||||
return List.of(
|
||||
Field.QUERY, Field.SCOPE, Field.FLOW_ID, Field.START_DATE, Field.END_DATE,
|
||||
Field.STATE, Field.LABELS, Field.TRIGGER_EXECUTION_ID, Field.CHILD_FILTER,
|
||||
Field.NAMESPACE,Field.KIND
|
||||
Field.NAMESPACE
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -256,25 +244,6 @@ public record QueryFilter(
|
||||
Field.START_DATE, Field.END_DATE, Field.TRIGGER_ID
|
||||
);
|
||||
}
|
||||
},
|
||||
SECRET_METADATA {
|
||||
@Override
|
||||
public List<Field> supportedField() {
|
||||
return List.of(
|
||||
Field.QUERY,
|
||||
Field.NAMESPACE
|
||||
);
|
||||
}
|
||||
},
|
||||
KV_METADATA {
|
||||
@Override
|
||||
public List<Field> supportedField() {
|
||||
return List.of(
|
||||
Field.QUERY,
|
||||
Field.NAMESPACE,
|
||||
Field.UPDATED
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
public abstract List<Field> supportedField();
|
||||
@@ -285,6 +254,18 @@ public record QueryFilter(
|
||||
*
|
||||
* @return List of {@code ResourceField} with resource names, fields, and operations.
|
||||
*/
|
||||
public static List<ResourceField> asResourceList() {
|
||||
return Arrays.stream(values())
|
||||
.map(Resource::toResourceField)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ResourceField toResourceField(Resource resource) {
|
||||
List<FieldOp> fieldOps = resource.supportedField().stream()
|
||||
.map(Resource::toFieldInfo)
|
||||
.toList();
|
||||
return new ResourceField(resource.name().toLowerCase(), fieldOps);
|
||||
}
|
||||
|
||||
private static FieldOp toFieldInfo(Field field) {
|
||||
List<Operation> operations = field.supportedOp().stream()
|
||||
@@ -298,6 +279,9 @@ public record QueryFilter(
|
||||
}
|
||||
}
|
||||
|
||||
public record ResourceField(String name, List<FieldOp> fields) {
|
||||
}
|
||||
|
||||
public record FieldOp(String name, String value, List<Operation> operations) {
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
package io.kestra.core.models;
|
||||
|
||||
public record TenantAndNamespace(String tenantId, String namespace) {}
|
||||
@@ -17,12 +17,31 @@ import java.util.List;
|
||||
@Introspected
|
||||
public class ExecutionUsage {
|
||||
private final List<DailyExecutionStatistics> dailyExecutionsCount;
|
||||
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
|
||||
|
||||
public static ExecutionUsage of(final String tenantId,
|
||||
final ExecutionRepositoryInterface executionRepository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
|
||||
try {
|
||||
dailyTaskRunsCount = executionRepository.dailyStatistics(
|
||||
null,
|
||||
tenantId,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null,
|
||||
true);
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
|
||||
}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(executionRepository.dailyStatistics(
|
||||
null,
|
||||
@@ -33,13 +52,28 @@ public class ExecutionUsage {
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
null))
|
||||
null,
|
||||
false))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
|
||||
final ZonedDateTime from,
|
||||
final ZonedDateTime to) {
|
||||
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
|
||||
try {
|
||||
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY,
|
||||
true
|
||||
);
|
||||
} catch (UnsupportedOperationException ignored) {}
|
||||
|
||||
return ExecutionUsage.builder()
|
||||
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
|
||||
null,
|
||||
@@ -47,8 +81,10 @@ public class ExecutionUsage {
|
||||
null,
|
||||
from,
|
||||
to,
|
||||
DateUtils.GroupType.DAY
|
||||
DateUtils.GroupType.DAY,
|
||||
false
|
||||
))
|
||||
.dailyTaskRunsCount(dailyTaskRunsCount)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,8 +32,6 @@ public class Dashboard implements HasUID, DeletedInterface {
|
||||
private String tenantId;
|
||||
|
||||
@Hidden
|
||||
@NotNull
|
||||
@NotBlank
|
||||
private String id;
|
||||
|
||||
@NotNull
|
||||
|
||||
@@ -5,6 +5,8 @@ import io.kestra.core.models.annotations.Plugin;
|
||||
import io.kestra.core.models.dashboards.filters.AbstractFilter;
|
||||
import io.kestra.core.repositories.QueryBuilderInterface;
|
||||
import io.kestra.plugin.core.dashboard.data.IData;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotBlank;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
@@ -36,6 +38,8 @@ public abstract class DataFilter<F extends Enum<F>, C extends ColumnDescriptor<F
|
||||
private Map<String, C> columns;
|
||||
|
||||
@Setter
|
||||
@Valid
|
||||
@Nullable
|
||||
private List<AbstractFilter<F>> where;
|
||||
|
||||
private List<OrderBy> orderBy;
|
||||
|
||||
@@ -500,7 +500,7 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
}
|
||||
|
||||
if (resolvedFinally != null && (
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailedNoRetry(resolvedTasks, parentTaskRun
|
||||
this.isTerminated(resolvedTasks, parentTaskRun) || this.hasFailed(resolvedTasks, parentTaskRun
|
||||
))) {
|
||||
return resolvedFinally;
|
||||
}
|
||||
@@ -588,13 +588,6 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastSubmitted(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
.filter(t -> t.getState().getCurrent() == State.Type.SUBMITTED)
|
||||
);
|
||||
}
|
||||
|
||||
public Optional<TaskRun> findLastRunning(List<TaskRun> taskRuns) {
|
||||
return Streams.findLast(taskRuns
|
||||
.stream()
|
||||
@@ -876,18 +869,20 @@ public class Execution implements DeletedInterface, TenantInterface {
|
||||
* @param e the exception raise
|
||||
* @return new taskRun with updated attempt with logs
|
||||
*/
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
|
||||
TaskRun failed = taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
);
|
||||
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
|
||||
TaskRunAttempt lastAttempt, Exception e) {
|
||||
return new FailedTaskRunWithLog(
|
||||
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
|
||||
taskRun
|
||||
.withAttempts(
|
||||
Stream
|
||||
.concat(
|
||||
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
|
||||
Stream.of(lastAttempt
|
||||
.withState(State.Type.FAILED))
|
||||
)
|
||||
.toList()
|
||||
)
|
||||
.withState(State.Type.FAILED),
|
||||
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import io.kestra.core.models.DeletedInterface;
|
||||
import io.kestra.core.models.TenantInterface;
|
||||
import io.kestra.core.models.executions.metrics.Counter;
|
||||
import io.kestra.core.models.executions.metrics.Gauge;
|
||||
import io.kestra.core.models.executions.metrics.Timer;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import jakarta.annotation.Nullable;
|
||||
@@ -83,10 +82,6 @@ public class MetricEntry implements DeletedInterface, TenantInterface {
|
||||
return counter.getValue();
|
||||
}
|
||||
|
||||
if (metricEntry instanceof Gauge gauge) {
|
||||
return gauge.getValue();
|
||||
}
|
||||
|
||||
if (metricEntry instanceof Timer timer) {
|
||||
return (double) timer.getValue().toMillis();
|
||||
}
|
||||
|
||||
@@ -197,17 +197,17 @@ public class TaskRun implements TenantInterface {
|
||||
taskRunBuilder.attempts = new ArrayList<>();
|
||||
|
||||
taskRunBuilder.attempts.add(TaskRunAttempt.builder()
|
||||
.state(new State(this.state, State.Type.RESUBMITTED))
|
||||
.state(new State(this.state, State.Type.KILLED))
|
||||
.build()
|
||||
);
|
||||
} else {
|
||||
ArrayList<TaskRunAttempt> taskRunAttempts = new ArrayList<>(taskRunBuilder.attempts);
|
||||
TaskRunAttempt lastAttempt = taskRunAttempts.get(taskRunBuilder.attempts.size() - 1);
|
||||
if (!lastAttempt.getState().isTerminated()) {
|
||||
taskRunAttempts.set(taskRunBuilder.attempts.size() - 1, lastAttempt.withState(State.Type.RESUBMITTED));
|
||||
taskRunAttempts.set(taskRunBuilder.attempts.size() - 1, lastAttempt.withState(State.Type.KILLED));
|
||||
} else {
|
||||
taskRunAttempts.add(TaskRunAttempt.builder()
|
||||
.state(new State().withState(State.Type.RESUBMITTED))
|
||||
.state(new State().withState(State.Type.KILLED))
|
||||
.build()
|
||||
);
|
||||
}
|
||||
@@ -301,7 +301,7 @@ public class TaskRun implements TenantInterface {
|
||||
}
|
||||
|
||||
public TaskRun incrementIteration() {
|
||||
int iteration = this.iteration == null ? 0 : this.iteration;
|
||||
int iteration = this.iteration == null ? 1 : this.iteration;
|
||||
return this.toBuilder()
|
||||
.iteration(iteration + 1)
|
||||
.build();
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package io.kestra.core.models.executions.metrics;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import jakarta.annotation.Nullable;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
import io.kestra.core.metrics.MetricRegistry;
|
||||
import io.kestra.core.models.executions.AbstractMetricEntry;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import java.util.Map;
|
||||
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class Gauge extends AbstractMetricEntry<Double> {
|
||||
public static final String TYPE = "gauge";
|
||||
|
||||
@NotNull
|
||||
@JsonInclude
|
||||
private final String type = TYPE;
|
||||
|
||||
@NotNull
|
||||
@EqualsAndHashCode.Exclude
|
||||
private Double value;
|
||||
|
||||
private Gauge(@NotNull String name, @Nullable String description, @NotNull Double value, String... tags) {
|
||||
super(name, description, tags);
|
||||
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @NotNull Double value, String... tags) {
|
||||
return new Gauge(name, null, value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @Nullable String description, @NotNull Double value, String... tags) {
|
||||
return new Gauge(name, description, value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @NotNull Integer value, String... tags) {
|
||||
return new Gauge(name, null, (double) value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @Nullable String description, @NotNull Integer value, String... tags) {
|
||||
return new Gauge(name, description, (double) value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @NotNull Long value, String... tags) {
|
||||
return new Gauge(name, null, (double) value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @Nullable String description, @NotNull Long value, String... tags) {
|
||||
return new Gauge(name, description, (double) value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @NotNull Float value, String... tags) {
|
||||
return new Gauge(name, null, (double) value, tags);
|
||||
}
|
||||
|
||||
public static Gauge of(@NotNull String name, @Nullable String description, @NotNull Float value, String... tags) {
|
||||
return new Gauge(name, description, (double) value, tags);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(MetricRegistry meterRegistry, String name, String description, Map<String, String> tags) {
|
||||
meterRegistry
|
||||
.gauge(this.metricName(name), description, this.value, this.tagsAsArray(tags));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void increment(Double value) {
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
@@ -49,7 +49,7 @@ import java.util.stream.Stream;
|
||||
public class Flow extends AbstractFlow implements HasUID {
|
||||
private static final ObjectMapper NON_DEFAULT_OBJECT_MAPPER = JacksonMapper.ofYaml()
|
||||
.copy()
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
|
||||
private static final ObjectMapper WITHOUT_REVISION_OBJECT_MAPPER = NON_DEFAULT_OBJECT_MAPPER.copy()
|
||||
.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
|
||||
|
||||
@@ -136,7 +136,7 @@ public interface FlowInterface extends FlowId, DeletedInterface, TenantInterface
|
||||
class SourceGenerator {
|
||||
private static final ObjectMapper NON_DEFAULT_OBJECT_MAPPER = JacksonMapper.ofJson()
|
||||
.copy()
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
|
||||
static String generate(final FlowInterface flow) {
|
||||
try {
|
||||
|
||||
@@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
import io.kestra.core.models.flows.input.*;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.validations.InputValidation;
|
||||
import io.micronaut.core.annotation.Introspected;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -45,7 +44,6 @@ import lombok.experimental.SuperBuilder;
|
||||
@JsonSubTypes.Type(value = YamlInput.class, name = "YAML"),
|
||||
@JsonSubTypes.Type(value = EmailInput.class, name = "EMAIL"),
|
||||
})
|
||||
@InputValidation
|
||||
public abstract class Input<T> implements Data {
|
||||
@Schema(
|
||||
title = "The ID of the input."
|
||||
@@ -82,13 +80,7 @@ public abstract class Input<T> implements Data {
|
||||
title = "The default value to use if no value is specified."
|
||||
)
|
||||
Property<T> defaults;
|
||||
|
||||
@Schema(
|
||||
title = "The suggested value for the input.",
|
||||
description = "Optional UI hint for pre-filling the input. Cannot be used together with a default value."
|
||||
)
|
||||
Property<T> prefill;
|
||||
|
||||
|
||||
@Schema(
|
||||
title = "The display name of the input."
|
||||
)
|
||||
|
||||
@@ -222,7 +222,6 @@ public class State {
|
||||
@Introspected
|
||||
public enum Type {
|
||||
CREATED,
|
||||
SUBMITTED,
|
||||
RUNNING,
|
||||
PAUSED,
|
||||
RESTARTED,
|
||||
@@ -236,15 +235,14 @@ public class State {
|
||||
RETRYING,
|
||||
RETRIED,
|
||||
SKIPPED,
|
||||
BREAKPOINT,
|
||||
RESUBMITTED;
|
||||
BREAKPOINT;
|
||||
|
||||
public boolean isTerminated() {
|
||||
return this == Type.FAILED || this == Type.WARNING || this == Type.SUCCESS || this == Type.KILLED || this == Type.CANCELLED || this == Type.RETRIED || this == Type.SKIPPED || this == Type.RESUBMITTED;
|
||||
return this == Type.FAILED || this == Type.WARNING || this == Type.SUCCESS || this == Type.KILLED || this == Type.CANCELLED || this == Type.RETRIED || this == Type.SKIPPED;
|
||||
}
|
||||
|
||||
public boolean isTerminatedNoFail() {
|
||||
return this == Type.WARNING || this == Type.SUCCESS || this == Type.RETRIED || this == Type.SKIPPED || this == Type.RESUBMITTED;
|
||||
return this == Type.WARNING || this == Type.SUCCESS || this == Type.RETRIED || this == Type.SKIPPED;
|
||||
}
|
||||
|
||||
public boolean isCreated() {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package io.kestra.core.models.flows.input;
|
||||
|
||||
import java.util.Set;
|
||||
import io.kestra.core.models.flows.Input;
|
||||
import io.kestra.core.validations.FileInputValidation;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -23,35 +22,10 @@ public class FileInput extends Input<URI> {
|
||||
|
||||
@Deprecated(since = "0.24", forRemoval = true)
|
||||
public String extension;
|
||||
|
||||
/**
|
||||
* List of allowed file extensions (e.g., [".csv", ".txt", ".pdf"]).
|
||||
* Each extension must start with a dot.
|
||||
*/
|
||||
private List<String> allowedFileExtensions;
|
||||
|
||||
/**
|
||||
* Gets the file extension from the URI's path
|
||||
*/
|
||||
private String getFileExtension(URI uri) {
|
||||
String path = uri.getPath();
|
||||
int lastDotIndex = path.lastIndexOf(".");
|
||||
return lastDotIndex >= 0 ? path.substring(lastDotIndex).toLowerCase() : "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(URI input) throws ConstraintViolationException {
|
||||
if (input == null || allowedFileExtensions == null || allowedFileExtensions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String extension = getFileExtension(input);
|
||||
if (!allowedFileExtensions.contains(extension.toLowerCase())) {
|
||||
throw new ConstraintViolationException(
|
||||
"File type not allowed. Accepted extensions: " + String.join(", ", allowedFileExtensions),
|
||||
Set.of()
|
||||
);
|
||||
}
|
||||
// no validation yet
|
||||
}
|
||||
|
||||
public static String findFileInputExtension(@NotNull final List<Input<?>> inputs, @NotNull final String fileName) {
|
||||
|
||||
@@ -8,7 +8,6 @@ import io.kestra.core.validations.Regex;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
@@ -28,7 +27,6 @@ public class SelectInput extends Input<String> implements RenderableInput {
|
||||
@Schema(
|
||||
title = "List of values."
|
||||
)
|
||||
@Size(min = 2)
|
||||
List<@Regex String> values;
|
||||
|
||||
@Schema(
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
package io.kestra.core.models.kv;
|
||||
|
||||
import io.kestra.core.models.DeletedInterface;
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.TenantInterface;
|
||||
import io.kestra.core.storages.kv.KVEntry;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.swagger.v3.oas.annotations.Hidden;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Pattern;
|
||||
import lombok.*;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Optional;
|
||||
|
||||
@Builder(toBuilder = true)
|
||||
@Slf4j
|
||||
@Getter
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
@AllArgsConstructor
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
public class PersistedKvMetadata implements DeletedInterface, TenantInterface, HasUID {
|
||||
@With
|
||||
@Hidden
|
||||
@Pattern(regexp = "^[a-z0-9][a-z0-9_-]*")
|
||||
private String tenantId;
|
||||
|
||||
@NotNull
|
||||
private String namespace;
|
||||
|
||||
@NotNull
|
||||
private String name;
|
||||
|
||||
private String description;
|
||||
|
||||
@NotNull
|
||||
private Integer version;
|
||||
|
||||
@Builder.Default
|
||||
private boolean last = true;
|
||||
|
||||
@Nullable
|
||||
private Instant expirationDate;
|
||||
|
||||
@Nullable
|
||||
private Instant created;
|
||||
|
||||
@Nullable
|
||||
private Instant updated;
|
||||
|
||||
private boolean deleted;
|
||||
|
||||
public static PersistedKvMetadata from(String tenantId, KVEntry kvEntry) {
|
||||
return PersistedKvMetadata.builder()
|
||||
.tenantId(tenantId)
|
||||
.namespace(kvEntry.namespace())
|
||||
.name(kvEntry.key())
|
||||
.version(kvEntry.version())
|
||||
.description(kvEntry.description())
|
||||
.created(kvEntry.creationDate())
|
||||
.updated(kvEntry.updateDate())
|
||||
.expirationDate(kvEntry.expirationDate())
|
||||
.build();
|
||||
}
|
||||
|
||||
public PersistedKvMetadata asLast() {
|
||||
Instant saveDate = Instant.now();
|
||||
return this.toBuilder().created(Optional.ofNullable(this.created).orElse(saveDate)).updated(saveDate).last(true).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String uid() {
|
||||
return IdUtils.fromParts(getTenantId(), getNamespace(), getName(), getVersion().toString());
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,7 @@ public class ResolvedTask {
|
||||
|
||||
public NextTaskRun toNextTaskRunIncrementIteration(Execution execution, Integer iteration) {
|
||||
return new NextTaskRun(
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 0),
|
||||
TaskRun.of(execution, this).withIteration(iteration != null ? iteration : 1),
|
||||
this.getTask()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ import java.util.Map;
|
||||
@JsonSubTypes({
|
||||
@JsonSubTypes.Type(value = CounterMetric.class, name = "counter"),
|
||||
@JsonSubTypes.Type(value = TimerMetric.class, name = "timer"),
|
||||
@JsonSubTypes.Type(value = GaugeMetric.class, name = "gauge"),
|
||||
})
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package io.kestra.core.models.tasks.metrics;
|
||||
|
||||
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.models.executions.AbstractMetricEntry;
|
||||
import io.kestra.core.models.executions.metrics.Gauge;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.*;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ToString
|
||||
@EqualsAndHashCode
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@SuperBuilder
|
||||
public class GaugeMetric extends AbstractMetric {
|
||||
public static final String TYPE = "gauge";
|
||||
|
||||
@NotNull
|
||||
@EqualsAndHashCode.Exclude
|
||||
private Property<Double> value;
|
||||
|
||||
@Override
|
||||
public AbstractMetricEntry<?> toMetric(RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
String name = runContext.render(this.name).as(String.class).orElseThrow();
|
||||
Double value = runContext.render(this.value).as(Double.class).orElseThrow();
|
||||
String description = runContext.render(this.description).as(String.class).orElse(null);
|
||||
Map<String, String> tags = runContext.render(this.tags).asMap(String.class, String.class);
|
||||
String[] tagsAsStrings = tags.entrySet().stream()
|
||||
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
|
||||
.toArray(String[]::new);
|
||||
|
||||
return Gauge.of(name, description, value, tagsAsStrings);
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return TYPE;
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,7 @@ public class Template implements DeletedInterface, TenantInterface, HasUID {
|
||||
return exclusions.contains(m.getName()) || super.hasIgnoreMarker(m);
|
||||
}
|
||||
})
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
|
||||
@Setter
|
||||
@Hidden
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package io.kestra.core.models.triggers;
|
||||
|
||||
import io.kestra.core.exceptions.InvalidTriggerConfigurationException;
|
||||
import io.kestra.core.models.annotations.PluginProperty;
|
||||
import io.kestra.core.models.conditions.ConditionContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import java.time.DateTimeException;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.Optional;
|
||||
@@ -31,29 +29,15 @@ public interface PollingTriggerInterface extends WorkerTriggerInterface {
|
||||
* Compute the next evaluation date of the trigger based on the existing trigger context: by default, it uses the current date and the interval.
|
||||
* Schedulable triggers must override this method.
|
||||
*/
|
||||
default ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optional<? extends TriggerContext> last) throws InvalidTriggerConfigurationException {
|
||||
return computeNextEvaluationDate();
|
||||
default ZonedDateTime nextEvaluationDate(ConditionContext conditionContext, Optional<? extends TriggerContext> last) throws Exception {
|
||||
return ZonedDateTime.now().plus(this.getInterval());
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the next evaluation date of the trigger: by default, it uses the current date and the interval.
|
||||
* Schedulable triggers must override this method as it's used to init them when there is no evaluation date.
|
||||
*/
|
||||
default ZonedDateTime nextEvaluationDate() throws InvalidTriggerConfigurationException {
|
||||
return computeNextEvaluationDate();
|
||||
}
|
||||
|
||||
/**
|
||||
* computes the next evaluation date using the configured interval.
|
||||
* Throw InvalidTriggerConfigurationException, if the interval causes date overflow.
|
||||
*/
|
||||
private ZonedDateTime computeNextEvaluationDate() throws InvalidTriggerConfigurationException {
|
||||
Duration interval = this.getInterval();
|
||||
|
||||
try {
|
||||
return ZonedDateTime.now().plus(interval);
|
||||
} catch (DateTimeException | ArithmeticException e) {
|
||||
throw new InvalidTriggerConfigurationException("Trigger interval too large", e);
|
||||
}
|
||||
default ZonedDateTime nextEvaluationDate() {
|
||||
return ZonedDateTime.now().plus(this.getInterval());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package io.kestra.core.models.triggers;
|
||||
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
public interface StatefulTriggerInterface {
|
||||
@Schema(
|
||||
title = "Trigger event type",
|
||||
description = """
|
||||
Defines when the trigger fires.
|
||||
- `CREATE`: only for newly discovered entities.
|
||||
- `UPDATE`: only when an already-seen entity changes.
|
||||
- `CREATE_OR_UPDATE`: fires on either event.
|
||||
"""
|
||||
)
|
||||
Property<On> getOn();
|
||||
|
||||
@Schema(
|
||||
title = "State key",
|
||||
description = """
|
||||
JSON-type KV key for persisted state.
|
||||
Default: `<namespace>__<flowId>__<triggerId>`
|
||||
"""
|
||||
)
|
||||
Property<String> getStateKey();
|
||||
|
||||
@Schema(
|
||||
title = "State TTL",
|
||||
description = "TTL for persisted state entries (e.g., PT24H, P7D)."
|
||||
)
|
||||
Property<Duration> getStateTtl();
|
||||
|
||||
enum On {
|
||||
CREATE,
|
||||
UPDATE,
|
||||
CREATE_OR_UPDATE
|
||||
}
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package io.kestra.core.models.triggers;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.kestra.core.storages.kv.KVMetadata;
|
||||
import io.kestra.core.storages.kv.KVValueAndMetadata;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class StatefulTriggerService {
|
||||
public record Entry(String uri, String version, Instant modifiedAt, Instant lastSeenAt) {
|
||||
public static Entry candidate(String uri, String version, Instant modifiedAt) {
|
||||
return new Entry(uri, version, modifiedAt, null);
|
||||
}
|
||||
}
|
||||
|
||||
public record StateUpdate(boolean fire, boolean isNew) {}
|
||||
|
||||
public static Map<String, Entry> readState(RunContext runContext, String key, Optional<Duration> ttl) {
|
||||
try {
|
||||
var kv = runContext.namespaceKv(runContext.flowInfo().namespace()).getValue(key);
|
||||
if (kv.isEmpty()) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
|
||||
var entries = JacksonMapper.ofJson().readValue((byte[]) kv.get().value(), new TypeReference<List<Entry>>() {});
|
||||
|
||||
var cutoff = ttl.map(d -> Instant.now().minus(d)).orElse(Instant.MIN);
|
||||
|
||||
return entries.stream()
|
||||
.filter(e -> Optional.ofNullable(e.lastSeenAt()).orElse(Instant.now()).isAfter(cutoff))
|
||||
.collect(Collectors.toMap(Entry::uri, e -> e));
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("readState failed", e);
|
||||
return new HashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
public static void writeState(RunContext runContext, String key, Map<String, Entry> state, Optional<Duration> ttl) {
|
||||
try {
|
||||
var bytes = JacksonMapper.ofJson().writeValueAsBytes(state.values());
|
||||
var meta = new KVMetadata("trigger state", ttl.orElse(null));
|
||||
|
||||
runContext.namespaceKv(runContext.flowInfo().namespace()).put(key, new KVValueAndMetadata(meta, bytes));
|
||||
} catch (Exception e) {
|
||||
runContext.logger().warn("writeState failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static StateUpdate computeAndUpdateState(Map<String, Entry> state, Entry candidate, StatefulTriggerInterface.On on) {
|
||||
var prev = state.get(candidate.uri());
|
||||
var isNew = prev == null;
|
||||
var fire = shouldFire(prev, candidate.version(), on);
|
||||
|
||||
Instant lastSeenAt;
|
||||
|
||||
if (fire || isNew) {
|
||||
// it is new seen or changed
|
||||
lastSeenAt = Instant.now();
|
||||
} else if (prev.lastSeenAt() != null) {
|
||||
// it is unchanged but already tracked before
|
||||
lastSeenAt = prev.lastSeenAt();
|
||||
} else {
|
||||
lastSeenAt = Instant.now();
|
||||
}
|
||||
|
||||
var newEntry = new Entry(candidate.uri(), candidate.version(), candidate.modifiedAt(), lastSeenAt);
|
||||
|
||||
state.put(candidate.uri(), newEntry);
|
||||
|
||||
return new StatefulTriggerService.StateUpdate(fire, isNew);
|
||||
}
|
||||
|
||||
public static boolean shouldFire(Entry prev, String version, StatefulTriggerInterface.On on) {
|
||||
if (prev == null) {
|
||||
return on == StatefulTriggerInterface.On.CREATE || on == StatefulTriggerInterface.On.CREATE_OR_UPDATE;
|
||||
}
|
||||
if (!Objects.equals(prev.version(), version)) {
|
||||
return on == StatefulTriggerInterface.On.UPDATE || on == StatefulTriggerInterface.On.CREATE_OR_UPDATE;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static String defaultKey(String ns, String flowId, String triggerId) {
|
||||
return String.join("_", ns, flowId, triggerId);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
package io.kestra.core.models.triggers;
|
||||
|
||||
import io.kestra.core.exceptions.InvalidTriggerConfigurationException;
|
||||
import io.kestra.core.models.HasUID;
|
||||
import io.kestra.core.models.conditions.ConditionContext;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
@@ -168,14 +167,9 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
// Used to update trigger in flowListeners
|
||||
public static Trigger of(FlowInterface flow, AbstractTrigger abstractTrigger, ConditionContext conditionContext, Optional<Trigger> lastTrigger) throws Exception {
|
||||
ZonedDateTime nextDate = null;
|
||||
boolean disabled = lastTrigger.map(TriggerContext::getDisabled).orElse(Boolean.FALSE);
|
||||
|
||||
if (abstractTrigger instanceof PollingTriggerInterface pollingTriggerInterface) {
|
||||
try {
|
||||
nextDate = pollingTriggerInterface.nextEvaluationDate(conditionContext, Optional.empty());
|
||||
} catch (InvalidTriggerConfigurationException e) {
|
||||
disabled = true;
|
||||
}
|
||||
nextDate = pollingTriggerInterface.nextEvaluationDate(conditionContext, Optional.empty());
|
||||
}
|
||||
|
||||
return Trigger.builder()
|
||||
@@ -186,7 +180,7 @@ public class Trigger extends TriggerContext implements HasUID {
|
||||
.date(ZonedDateTime.now().truncatedTo(ChronoUnit.SECONDS))
|
||||
.nextExecutionDate(nextDate)
|
||||
.stopAfter(abstractTrigger.getStopAfter())
|
||||
.disabled(disabled)
|
||||
.disabled(lastTrigger.map(TriggerContext::getDisabled).orElse(Boolean.FALSE))
|
||||
.backfill(null)
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import io.kestra.core.runners.FlowInputOutput;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.utils.IdUtils;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
|
||||
@@ -24,7 +25,7 @@ public abstract class TriggerService {
|
||||
RunContext runContext = conditionContext.getRunContext();
|
||||
ExecutionTrigger executionTrigger = ExecutionTrigger.of(trigger, variables, runContext.logFileURI());
|
||||
|
||||
return generateExecution(runContext.getTriggerExecutionId(), trigger, context, executionTrigger, conditionContext);
|
||||
return generateExecution(runContext.getTriggerExecutionId(), trigger, context, executionTrigger, conditionContext.getFlow().getRevision());
|
||||
}
|
||||
|
||||
public static Execution generateExecution(
|
||||
@@ -36,7 +37,7 @@ public abstract class TriggerService {
|
||||
RunContext runContext = conditionContext.getRunContext();
|
||||
ExecutionTrigger executionTrigger = ExecutionTrigger.of(trigger, output, runContext.logFileURI());
|
||||
|
||||
return generateExecution(runContext.getTriggerExecutionId(), trigger, context, executionTrigger, conditionContext);
|
||||
return generateExecution(runContext.getTriggerExecutionId(), trigger, context, executionTrigger, conditionContext.getFlow().getRevision());
|
||||
}
|
||||
|
||||
public static Execution generateRealtimeExecution(
|
||||
@@ -48,7 +49,7 @@ public abstract class TriggerService {
|
||||
RunContext runContext = conditionContext.getRunContext();
|
||||
ExecutionTrigger executionTrigger = ExecutionTrigger.of(trigger, output, runContext.logFileURI());
|
||||
|
||||
return generateExecution(IdUtils.create(), trigger, context, executionTrigger, conditionContext);
|
||||
return generateExecution(IdUtils.create(), trigger, context, executionTrigger, conditionContext.getFlow().getRevision());
|
||||
}
|
||||
|
||||
public static Execution generateScheduledExecution(
|
||||
@@ -74,7 +75,6 @@ public abstract class TriggerService {
|
||||
.namespace(context.getNamespace())
|
||||
.flowId(context.getFlowId())
|
||||
.flowRevision(conditionContext.getFlow().getRevision())
|
||||
.variables(conditionContext.getFlow().getVariables())
|
||||
.labels(executionLabels)
|
||||
.state(new State())
|
||||
.trigger(executionTrigger)
|
||||
@@ -108,7 +108,7 @@ public abstract class TriggerService {
|
||||
AbstractTrigger trigger,
|
||||
TriggerContext context,
|
||||
ExecutionTrigger executionTrigger,
|
||||
ConditionContext conditionContext
|
||||
Integer flowRevision
|
||||
) {
|
||||
List<Label> executionLabels = new ArrayList<>(ListUtils.emptyOnNull(trigger.getLabels()));
|
||||
if (executionLabels.stream().noneMatch(label -> Label.CORRELATION_ID.equals(label.key()))) {
|
||||
@@ -120,8 +120,7 @@ public abstract class TriggerService {
|
||||
.namespace(context.getNamespace())
|
||||
.flowId(context.getFlowId())
|
||||
.tenantId(context.getTenantId())
|
||||
.flowRevision(conditionContext.getFlow().getRevision())
|
||||
.variables(conditionContext.getFlow().getVariables())
|
||||
.flowRevision(flowRevision)
|
||||
.state(new State())
|
||||
.trigger(executionTrigger)
|
||||
.labels(executionLabels)
|
||||
|
||||
@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
*
|
||||
* @return the {@link DefaultPluginRegistry}.
|
||||
*/
|
||||
public synchronized static DefaultPluginRegistry getOrCreate() {
|
||||
public static DefaultPluginRegistry getOrCreate() {
|
||||
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
|
||||
if (!instance.isInitialized()) {
|
||||
instance.init();
|
||||
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
/**
|
||||
* Initializes the registry by loading all core plugins.
|
||||
*/
|
||||
protected synchronized void init() {
|
||||
protected void init() {
|
||||
if (initialized.compareAndSet(false, true)) {
|
||||
register(scanner.scan());
|
||||
}
|
||||
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
if (existing != null && existing.crc32() == plugin.crc32()) {
|
||||
return; // same plugin already registered
|
||||
}
|
||||
|
||||
|
||||
lock.lock();
|
||||
try {
|
||||
if (existing != null) {
|
||||
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
|
||||
pluginClassByIdentifier.putAll(plugins);
|
||||
}
|
||||
|
||||
@@ -14,19 +14,19 @@ import java.time.Instant;
|
||||
@Requires(property = "kestra.server-type")
|
||||
@Slf4j
|
||||
public class ReportableScheduler {
|
||||
|
||||
|
||||
private final ReportableRegistry registry;
|
||||
private final ServerEventSender sender;
|
||||
private final Clock clock;
|
||||
|
||||
|
||||
@Inject
|
||||
public ReportableScheduler(ReportableRegistry registry, ServerEventSender sender) {
|
||||
this.registry = registry;
|
||||
this.sender = sender;
|
||||
this.clock = Clock.systemDefaultZone();
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = "5m", initialDelay = "${kestra.anonymous-usage-report.initial-delay:5m}")
|
||||
|
||||
@Scheduled(fixedDelay = "5m", initialDelay = "${kestra.anonymous-usage-report.initial-delay}")
|
||||
public void tick() {
|
||||
Instant now = clock.instant();
|
||||
for (Reportable<?> r : registry.getAll()) {
|
||||
|
||||
@@ -18,7 +18,6 @@ import io.micronaut.http.hateoas.JsonError;
|
||||
import io.micronaut.reactor.http.client.ReactorHttpClient;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.net.URI;
|
||||
@@ -29,30 +28,29 @@ import java.util.UUID;
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class ServerEventSender {
|
||||
|
||||
|
||||
private static final String SESSION_UUID = IdUtils.create();
|
||||
private static final ObjectMapper OBJECT_MAPPER = JacksonMapper.ofJson();
|
||||
|
||||
|
||||
@Inject
|
||||
@Client
|
||||
private ReactorHttpClient client;
|
||||
|
||||
|
||||
@Inject
|
||||
private VersionProvider versionProvider;
|
||||
|
||||
|
||||
@Inject
|
||||
private InstanceService instanceService;
|
||||
|
||||
|
||||
private final ServerType serverType;
|
||||
|
||||
@Setter
|
||||
@Value("${kestra.anonymous-usage-report.uri:'https://api.kestra.io/v1/reports/server-events'}")
|
||||
|
||||
@Value("${kestra.anonymous-usage-report.uri}")
|
||||
protected URI url;
|
||||
|
||||
|
||||
public ServerEventSender( ) {
|
||||
this.serverType = KestraContext.getContext().getServerType();
|
||||
}
|
||||
|
||||
|
||||
public void send(final Instant now, final Type type, Object event) {
|
||||
ServerEvent serverEvent = ServerEvent
|
||||
.builder()
|
||||
@@ -67,11 +65,11 @@ public class ServerEventSender {
|
||||
.build();
|
||||
try {
|
||||
MutableHttpRequest<ServerEvent> request = this.request(serverEvent, type);
|
||||
|
||||
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("Report anonymous usage: '{}'", OBJECT_MAPPER.writeValueAsString(serverEvent));
|
||||
}
|
||||
|
||||
|
||||
this.handleResponse(client.toBlocking().retrieve(request, Argument.of(Result.class), Argument.of(JsonError.class)));
|
||||
} catch (HttpClientResponseException t) {
|
||||
log.trace("Unable to report anonymous usage with body '{}'", t.getResponse().getBody(String.class), t);
|
||||
@@ -79,11 +77,11 @@ public class ServerEventSender {
|
||||
log.trace("Unable to handle anonymous usage", t);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void handleResponse (Result result){
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
protected MutableHttpRequest<ServerEvent> request(ServerEvent event, Type type) throws Exception {
|
||||
URI baseUri = URI.create(this.url.toString().endsWith("/") ? this.url.toString() : this.url + "/");
|
||||
URI resolvedUri = baseUri.resolve(type.name().toLowerCase());
|
||||
|
||||
@@ -25,6 +25,8 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
|
||||
Boolean isTaskRunEnabled();
|
||||
|
||||
default Optional<Execution> findById(String tenantId, String id) {
|
||||
return findById(tenantId, id, false);
|
||||
}
|
||||
@@ -94,6 +96,12 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
|
||||
Flux<Execution> findAllAsync(@Nullable String tenantId);
|
||||
|
||||
ArrayListTotal<TaskRun> findTaskRun(
|
||||
Pageable pageable,
|
||||
@Nullable String tenantId,
|
||||
List<QueryFilter> filters
|
||||
);
|
||||
|
||||
Execution delete(Execution execution);
|
||||
|
||||
Integer purge(Execution execution);
|
||||
@@ -106,7 +114,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable String flowId,
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
List<DailyExecutionStatistics> dailyStatistics(
|
||||
@@ -118,7 +127,8 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
|
||||
@Nullable ZonedDateTime startDate,
|
||||
@Nullable ZonedDateTime endDate,
|
||||
@Nullable DateUtils.GroupType groupBy,
|
||||
List<State.Type> state
|
||||
List<State.Type> state,
|
||||
boolean isTaskRun
|
||||
);
|
||||
|
||||
@Getter
|
||||
|
||||
@@ -4,7 +4,6 @@ import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.SearchResult;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.plugin.core.dashboard.data.Flows;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.validation.ConstraintViolationException;
|
||||
@@ -12,7 +11,7 @@ import jakarta.validation.ConstraintViolationException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fields> {
|
||||
public interface FlowRepositoryInterface {
|
||||
|
||||
Optional<Flow> findById(String tenantId, String namespace, String id, Optional<Integer> revision, Boolean allowDeleted);
|
||||
|
||||
@@ -163,6 +162,4 @@ public interface FlowRepositoryInterface extends QueryBuilderInterface<Flows.Fie
|
||||
FlowWithSource update(GenericFlow flow, FlowInterface previous) throws ConstraintViolationException;
|
||||
|
||||
FlowWithSource delete(FlowInterface flow);
|
||||
|
||||
Boolean existAnyNoAcl(String tenantId);
|
||||
}
|
||||
|
||||
@@ -10,8 +10,6 @@ public interface FlowTopologyRepositoryInterface {
|
||||
|
||||
List<FlowTopology> findByNamespace(String tenantId, String namespace);
|
||||
|
||||
List<FlowTopology> findByNamespacePrefix(String tenantId, String namespacePrefix);
|
||||
|
||||
List<FlowTopology> findAll(String tenantId);
|
||||
|
||||
FlowTopology save(FlowTopology flowTopology);
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
package io.kestra.core.repositories;
|
||||
|
||||
import io.kestra.core.models.FetchVersion;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface KvMetadataRepositoryInterface extends SaveRepositoryInterface<PersistedKvMetadata> {
|
||||
Optional<PersistedKvMetadata> findByName(
|
||||
String tenantId,
|
||||
String namespace,
|
||||
String name
|
||||
) throws IOException;
|
||||
|
||||
default ArrayListTotal<PersistedKvMetadata> find(
|
||||
Pageable pageable,
|
||||
String tenantId,
|
||||
List<QueryFilter> filters,
|
||||
boolean allowDeleted,
|
||||
boolean allowExpired
|
||||
) {
|
||||
return this.find(pageable, tenantId, filters, allowDeleted, allowExpired, FetchVersion.LATEST);
|
||||
}
|
||||
|
||||
ArrayListTotal<PersistedKvMetadata> find(
|
||||
Pageable pageable,
|
||||
String tenantId,
|
||||
List<QueryFilter> filters,
|
||||
boolean allowDeleted,
|
||||
boolean allowExpired,
|
||||
FetchVersion fetchBehavior
|
||||
);
|
||||
|
||||
default PersistedKvMetadata delete(PersistedKvMetadata persistedKvMetadata) throws IOException {
|
||||
return this.save(persistedKvMetadata.toBuilder().deleted(true).build());
|
||||
}
|
||||
|
||||
/**
|
||||
* Purge (hard delete) a list of persisted kv metadata. If no version is specified, all versions are purged.
|
||||
* @param persistedKvsMetadata the list of persisted kv metadata to purge
|
||||
* @return the number of purged persisted kv metadata
|
||||
*/
|
||||
Integer purge(List<PersistedKvMetadata> persistedKvsMetadata);
|
||||
}
|
||||
@@ -83,9 +83,7 @@ public class LocalFlowRepositoryLoader {
|
||||
}
|
||||
|
||||
public void load(String tenantId, File basePath) throws IOException {
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
|
||||
.stream()
|
||||
.filter(flow -> tenantId.equals(flow.getTenantId()))
|
||||
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
|
||||
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
|
||||
|
||||
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {
|
||||
|
||||
@@ -5,7 +5,10 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
|
||||
import io.kestra.core.exceptions.InternalException;
|
||||
import io.kestra.core.models.Label;
|
||||
import io.kestra.core.models.executions.*;
|
||||
import io.kestra.core.models.flows.*;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowInterface;
|
||||
import io.kestra.core.models.flows.FlowWithException;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.models.property.Property;
|
||||
import io.kestra.core.models.tasks.ExecutableTask;
|
||||
import io.kestra.core.models.tasks.Task;
|
||||
@@ -26,7 +29,6 @@ import org.apache.commons.lang3.stream.Streams;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.kestra.core.trace.Tracer.throwCallable;
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
@@ -151,22 +153,14 @@ public final class ExecutableUtils {
|
||||
currentFlow.getNamespace(),
|
||||
currentFlow.getId()
|
||||
)
|
||||
.orElseThrow(() -> {
|
||||
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
|
||||
runContext.logger().error(msg);
|
||||
return new IllegalStateException(msg);
|
||||
});
|
||||
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
|
||||
|
||||
if (flow.isDisabled()) {
|
||||
String msg = "Cannot execute a flow which is disabled";
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute a flow which is disabled");
|
||||
}
|
||||
|
||||
if (flow instanceof FlowWithException fwe) {
|
||||
String msg = "Cannot execute an invalid flow: " + fwe.getException();
|
||||
runContext.logger().error(msg);
|
||||
throw new IllegalStateException(msg);
|
||||
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
|
||||
}
|
||||
|
||||
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
|
||||
@@ -207,20 +201,7 @@ public final class ExecutableUtils {
|
||||
.build()
|
||||
)
|
||||
.withScheduleDate(scheduleOnDate);
|
||||
if(execution.getInputs().size()<inputs.size()) {
|
||||
Map<String,Object>resolvedInputs=execution.getInputs();
|
||||
for (var inputKey : inputs.keySet()) {
|
||||
if (!resolvedInputs.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
|
||||
inputKey,
|
||||
currentExecution.getId(),
|
||||
currentTask.subflowId().namespace(),
|
||||
currentTask.subflowId().flowId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// inject the traceparent into the new execution
|
||||
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));
|
||||
|
||||
|
||||
@@ -82,8 +82,7 @@ public abstract class FilesService {
|
||||
}
|
||||
|
||||
private static String resolveUniqueNameForFile(final Path path) {
|
||||
String filename = path.getFileName().toString();
|
||||
String encodedFilename = java.net.URLEncoder.encode(filename, java.nio.charset.StandardCharsets.UTF_8);
|
||||
return IdUtils.from(path.toString()) + "-" + encodedFilename;
|
||||
String filename = path.getFileName().toString().replace(' ', '+');
|
||||
return IdUtils.from(path.toString()) + "-" + filename;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,15 @@ import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.util.*;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -130,7 +138,7 @@ public class FlowInputOutput {
|
||||
private Mono<Map<String, Object>> readData(List<Input<?>> inputs, Execution execution, Publisher<CompletedPart> data, boolean uploadFiles) {
|
||||
return Flux.from(data)
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.<Map.Entry<String, String>>handle((input, sink) -> {
|
||||
.<AbstractMap.SimpleEntry<String, String>>handle((input, sink) -> {
|
||||
if (input instanceof CompletedFileUpload fileUpload) {
|
||||
boolean oldStyleInput = false;
|
||||
if ("files".equals(fileUpload.getName())) {
|
||||
@@ -150,7 +158,7 @@ public class FlowInputOutput {
|
||||
.getContextStorageURI()
|
||||
);
|
||||
fileUpload.discard();
|
||||
sink.next(Map.entry(inputId, from.toString()));
|
||||
sink.next(new AbstractMap.SimpleEntry<>(inputId, from.toString()));
|
||||
} else {
|
||||
try {
|
||||
final String fileExtension = FileInput.findFileInputExtension(inputs, fileName);
|
||||
@@ -165,7 +173,7 @@ public class FlowInputOutput {
|
||||
return;
|
||||
}
|
||||
URI from = storageInterface.from(execution, inputId, fileName, tempFile);
|
||||
sink.next(Map.entry(inputId, from.toString()));
|
||||
sink.next(new AbstractMap.SimpleEntry<>(inputId, from.toString()));
|
||||
} finally {
|
||||
if (!tempFile.delete()) {
|
||||
tempFile.deleteOnExit();
|
||||
@@ -178,13 +186,13 @@ public class FlowInputOutput {
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
sink.next(Map.entry(input.getName(), new String(input.getBytes())));
|
||||
sink.next(new AbstractMap.SimpleEntry<>(input.getName(), new String(input.getBytes())));
|
||||
} catch (IOException e) {
|
||||
sink.error(e);
|
||||
}
|
||||
}
|
||||
})
|
||||
.collectMap(Map.Entry::getKey, Map.Entry::getValue);
|
||||
.collectMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -220,19 +228,6 @@ public class FlowInputOutput {
|
||||
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
|
||||
})
|
||||
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
|
||||
if (resolved.size() < data.size()) {
|
||||
RunContext runContext = runContextFactory.of(flow, execution);
|
||||
for (var inputKey : data.keySet()) {
|
||||
if (!resolved.containsKey(inputKey)) {
|
||||
runContext.logger().warn(
|
||||
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
|
||||
inputKey,
|
||||
flow.getNamespace(),
|
||||
flow.getId()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return MapUtils.flattenToNestedMap(resolved);
|
||||
}
|
||||
|
||||
@@ -327,14 +322,14 @@ public class FlowInputOutput {
|
||||
resolvable.setInput(input);
|
||||
|
||||
Object value = resolvable.get().value();
|
||||
|
||||
|
||||
// resolve default if needed
|
||||
if (value == null && input.getDefaults() != null) {
|
||||
RunContext runContextForDefault = decryptSecrets ? runContext : buildRunContextForExecutionAndInputs(flow, execution, dependencies, false);
|
||||
value = resolveDefaultValue(input, runContextForDefault);
|
||||
resolvable.isDefault(true);
|
||||
}
|
||||
|
||||
|
||||
// validate and parse input value
|
||||
if (value == null) {
|
||||
if (input.getRequired()) {
|
||||
@@ -363,7 +358,7 @@ public class FlowInputOutput {
|
||||
|
||||
return resolvable.get();
|
||||
}
|
||||
|
||||
|
||||
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
|
||||
return switch (input.getType()) {
|
||||
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
|
||||
@@ -380,7 +375,7 @@ public class FlowInputOutput {
|
||||
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
|
||||
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
|
||||
@@ -466,7 +461,7 @@ public class FlowInputOutput {
|
||||
if (data.getType() == null) {
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
|
||||
}
|
||||
|
||||
|
||||
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
|
||||
|
||||
return Optional.of(new AbstractMap.SimpleEntry<>(
|
||||
@@ -543,17 +538,17 @@ public class FlowInputOutput {
|
||||
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
|
||||
if (outputs == null) return Map.of();
|
||||
|
||||
|
||||
// render required outputs
|
||||
Map<String, Object> outputsById = outputs
|
||||
.stream()
|
||||
.filter(output -> output.getRequired() == null || output.getRequired())
|
||||
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
|
||||
outputsById = runContext.render(outputsById);
|
||||
|
||||
|
||||
// render optional outputs one by one to catch, log, and skip any error.
|
||||
for (io.kestra.core.models.flows.Output output : outputs) {
|
||||
if (Boolean.FALSE.equals(output.getRequired())) {
|
||||
@@ -596,9 +591,9 @@ public class FlowInputOutput {
|
||||
}
|
||||
|
||||
public void isDefault(boolean isDefault) {
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
|
||||
}
|
||||
|
||||
|
||||
public void setInput(final Input<?> input) {
|
||||
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
|
||||
}
|
||||
|
||||
@@ -84,12 +84,6 @@ public class FlowableUtils {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have submitted, leave
|
||||
Optional<TaskRun> lastSubmitted = execution.findLastSubmitted(taskRuns);
|
||||
if (lastSubmitted.isPresent()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
// have running, leave
|
||||
Optional<TaskRun> lastRunning = execution.findLastRunning(taskRuns);
|
||||
if (lastRunning.isPresent()) {
|
||||
|
||||
@@ -176,10 +176,6 @@ public abstract class RunContext implements PropertyContext {
|
||||
*/
|
||||
public abstract KVStore namespaceKv(String namespace);
|
||||
|
||||
/**
|
||||
* @deprecated use #namespaceKv(String) instead
|
||||
*/
|
||||
@Deprecated(since = "1.1.0", forRemoval = true)
|
||||
public StateStore stateStore() {
|
||||
return new StateStore(this, true);
|
||||
}
|
||||
|
||||
@@ -168,7 +168,6 @@ public class Extension extends AbstractExtension {
|
||||
functions.put("randomPort", new RandomPortFunction());
|
||||
functions.put("fileExists", fileExistsFunction);
|
||||
functions.put("isFileEmpty", isFileEmptyFunction);
|
||||
functions.put("nanoId", new NanoIDFunction());
|
||||
functions.put("tasksWithState", new TasksWithStateFunction());
|
||||
functions.put(HttpFunction.NAME, httpFunction);
|
||||
return functions;
|
||||
|
||||
@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
|
||||
ZoneId zoneId = zoneId(timeZone);
|
||||
ZonedDateTime date = convert(input, zoneId, existingFormat);
|
||||
|
||||
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,10 +151,7 @@ abstract class AbstractFileFunction implements Function {
|
||||
// if there is a trigger of type execution, we also allow accessing a file from the parent execution
|
||||
Map<String, String> trigger = (Map<String, String>) context.getVariable(TRIGGER);
|
||||
|
||||
if (!isFileUriValid(trigger.get(NAMESPACE), trigger.get("flowId"), trigger.get("executionId"), path)) {
|
||||
throw new IllegalArgumentException("Unable to read the file '" + path + "' as it didn't belong to the parent execution");
|
||||
}
|
||||
return true;
|
||||
return isFileUriValid(trigger.get(NAMESPACE), trigger.get("flowId"), trigger.get("executionId"), path);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import io.kestra.core.http.HttpRequest;
|
||||
import io.kestra.core.http.HttpResponse;
|
||||
import io.kestra.core.http.client.HttpClient;
|
||||
import io.kestra.core.http.client.HttpClientException;
|
||||
import io.kestra.core.http.client.HttpClientRequestException;
|
||||
import io.kestra.core.http.client.HttpClientResponseException;
|
||||
import io.kestra.core.http.client.configurations.HttpConfiguration;
|
||||
import io.kestra.core.runners.RunContext;
|
||||
import io.kestra.core.runners.RunContextFactory;
|
||||
@@ -103,15 +101,8 @@ public class HttpFunction<T> implements Function {
|
||||
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
|
||||
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
|
||||
return response.getBody();
|
||||
} catch (HttpClientResponseException e) {
|
||||
if (e.getResponse() != null) {
|
||||
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
|
||||
throw new PebbleException(e, msg , lineNumber, self.getName());
|
||||
} else {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
}
|
||||
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
|
||||
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
|
||||
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
|
||||
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package io.kestra.core.runners.pebble.functions;
|
||||
|
||||
import io.pebbletemplates.pebble.error.PebbleException;
|
||||
import io.pebbletemplates.pebble.extension.Function;
|
||||
import io.pebbletemplates.pebble.template.EvaluationContext;
|
||||
import io.pebbletemplates.pebble.template.PebbleTemplate;
|
||||
|
||||
import java.security.SecureRandom;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class NanoIDFunction implements Function {
|
||||
|
||||
private static final int DEFAULT_LENGTH = 21;
|
||||
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
private static final String LENGTH = "length";
|
||||
private static final String ALPHABET = "alphabet";
|
||||
|
||||
private static final int MAX_LENGTH = 1000;
|
||||
|
||||
@Override
|
||||
public Object execute(
|
||||
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
|
||||
int length = DEFAULT_LENGTH;
|
||||
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
|
||||
length = parseLength(args, self, lineNumber);
|
||||
}
|
||||
char[] alphabet = DEFAULT_ALPHABET;
|
||||
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
|
||||
alphabet = ((String) args.get(ALPHABET)).toCharArray();
|
||||
}
|
||||
return createNanoID(length, alphabet);
|
||||
}
|
||||
|
||||
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
|
||||
var value = (Long) args.get(LENGTH);
|
||||
if(value > MAX_LENGTH) {
|
||||
throw new PebbleException(
|
||||
null,
|
||||
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
|
||||
lineNumber,
|
||||
self.getName());
|
||||
}
|
||||
return Math.toIntExact(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getArgumentNames() {
|
||||
return List.of(LENGTH,ALPHABET);
|
||||
}
|
||||
|
||||
String createNanoID(int length, char[] alphabet){
|
||||
final char[] data = new char[length];
|
||||
final byte[] bytes = new byte[length];
|
||||
final int mask = alphabet.length-1;
|
||||
secureRandom.nextBytes(bytes);
|
||||
for (int i = 0; i < length; ++i) {
|
||||
data[i] = alphabet[bytes[i] & mask];
|
||||
}
|
||||
return String.valueOf(data);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,29 +1,21 @@
|
||||
package io.kestra.core.secret;
|
||||
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.repositories.ArrayListTotal;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.Strings;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class SecretService<META> {
|
||||
public class SecretService {
|
||||
private static final String SECRET_PREFIX = "SECRET_";
|
||||
|
||||
private Map<String, String> decodedSecrets;
|
||||
|
||||
|
||||
@PostConstruct
|
||||
private void postConstruct() {
|
||||
this.decode();
|
||||
@@ -54,28 +46,6 @@ public class SecretService<META> {
|
||||
return secret;
|
||||
}
|
||||
|
||||
public ArrayListTotal<META> list(Pageable pageable, String tenantId, List<QueryFilter> filters) throws IOException {
|
||||
final Predicate<String> queryPredicate = filters.stream()
|
||||
.filter(filter -> filter.field().equals(QueryFilter.Field.QUERY) && filter.value() != null)
|
||||
.findFirst()
|
||||
.map(filter -> {
|
||||
if (filter.operation().equals(QueryFilter.Op.EQUALS)) {
|
||||
return (Predicate<String>) s -> Strings.CI.contains(s, (String) filter.value());
|
||||
} else if (filter.operation().equals(QueryFilter.Op.NOT_EQUALS)) {
|
||||
return (Predicate<String>) s -> !Strings.CI.contains(s, (String) filter.value());
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported operation for QUERY filter: " + filter.operation());
|
||||
}
|
||||
})
|
||||
.orElse(s -> true);
|
||||
|
||||
//noinspection unchecked
|
||||
return ArrayListTotal.of(
|
||||
pageable,
|
||||
decodedSecrets.keySet().stream().filter(queryPredicate).map(s -> (META) s).toList()
|
||||
);
|
||||
}
|
||||
|
||||
public Map<String, Set<String>> inheritedSecrets(String tenantId, String namespace) throws IOException {
|
||||
return Map.of(namespace, decodedSecrets.keySet());
|
||||
}
|
||||
|
||||
@@ -180,13 +180,23 @@ public final class FileSerde {
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
|
||||
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
|
||||
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
|
||||
// and a sequence/stream of things (of Array shape).
|
||||
// To work around that, we need to create a JsonParser and advance to the first token.
|
||||
try (var parser = objectMapper.createParser(reader)) {
|
||||
parser.nextToken();
|
||||
return objectMapper.readerFor(type).readValues(parser);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ public final class JacksonMapper {
|
||||
|
||||
return mapper
|
||||
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL)
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
|
||||
.registerModule(new JavaTimeModule())
|
||||
.registerModule(new Jdk8Module())
|
||||
.registerModule(new ParameterNamesModule())
|
||||
@@ -153,7 +153,7 @@ public final class JacksonMapper {
|
||||
|
||||
private static ObjectMapper createIonObjectMapper() {
|
||||
return configure(new IonObjectMapper(new IonFactory(createIonSystem())))
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.ALWAYS)
|
||||
.setSerializationInclusion(JsonInclude.Include.ALWAYS)
|
||||
.registerModule(new IonModule());
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ public final class JacksonMapper {
|
||||
|
||||
return Pair.of(patch, revert);
|
||||
}
|
||||
|
||||
|
||||
public static JsonNode applyPatchesOnJsonNode(JsonNode jsonObject, List<JsonNode> patches) {
|
||||
for (JsonNode patch : patches) {
|
||||
try {
|
||||
|
||||
@@ -3,21 +3,15 @@ package io.kestra.core.services;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.flows.State;
|
||||
import io.kestra.core.queues.QueueException;
|
||||
import io.kestra.core.runners.ConcurrencyLimit;
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Contains methods to manage concurrency limit.
|
||||
* This is designed to be used by the API, the executor use lower level primitives.
|
||||
*/
|
||||
public interface ConcurrencyLimitService {
|
||||
@Singleton
|
||||
public class ConcurrencyLimitService {
|
||||
|
||||
Set<State.Type> VALID_TARGET_STATES =
|
||||
protected static final Set<State.Type> VALID_TARGET_STATES =
|
||||
EnumSet.of(State.Type.RUNNING, State.Type.CANCELLED, State.Type.FAILED);
|
||||
|
||||
/**
|
||||
@@ -25,20 +19,18 @@ public interface ConcurrencyLimitService {
|
||||
*
|
||||
* @throws IllegalArgumentException in case the execution is not queued.
|
||||
*/
|
||||
Execution unqueue(Execution execution, State.Type state) throws QueueException;
|
||||
public Execution unqueue(Execution execution, State.Type state) throws QueueException {
|
||||
if (execution.getState().getCurrent() != State.Type.QUEUED) {
|
||||
throw new IllegalArgumentException("Only QUEUED execution can be unqueued");
|
||||
}
|
||||
|
||||
/**
|
||||
* Find concurrency limits.
|
||||
*/
|
||||
List<ConcurrencyLimit> find(String tenantId);
|
||||
state = (state == null) ? State.Type.RUNNING : state;
|
||||
|
||||
/**
|
||||
* Update a concurrency limit.
|
||||
*/
|
||||
ConcurrencyLimit update(ConcurrencyLimit concurrencyLimit);
|
||||
// Validate the target state, throwing an exception if the state is invalid
|
||||
if (!VALID_TARGET_STATES.contains(state)) {
|
||||
throw new IllegalArgumentException("Invalid target state: " + state + ". Valid states are: " + VALID_TARGET_STATES);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a concurrency limit by its identifier.
|
||||
*/
|
||||
Optional<ConcurrencyLimit> findById(String tenantId, String namespace, String flowId);
|
||||
return execution.withState(state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -737,8 +737,8 @@ public class ExecutionService {
|
||||
} else {
|
||||
newExecution = execution.withState(killingOrAfterKillState);
|
||||
}
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
|
||||
// Because this method is expected to be called by the Executor we can return the Execution
|
||||
// immediately without publishing a CrudEvent like it's done on pause/resume method.
|
||||
return newExecution;
|
||||
}
|
||||
|
||||
@@ -285,10 +285,6 @@ public class FlowService {
|
||||
if ((subflowId != null && subflowId.matches(regex)) || (namespace != null && namespace.matches(regex))) {
|
||||
return;
|
||||
}
|
||||
if (subflowId == null || namespace == null) {
|
||||
// those fields are mandatory so the mandatory validation will apply
|
||||
return;
|
||||
}
|
||||
Optional<Flow> optional = findById(tenantId, subflow.getNamespace(), subflow.getFlowId());
|
||||
|
||||
if (optional.isEmpty()) {
|
||||
@@ -548,8 +544,6 @@ public class FlowService {
|
||||
|
||||
var flowTopologies = flowTopologyRepository.get().findByFlow(tenantId, namespace, id, destinationOnly);
|
||||
|
||||
var visitedNodes = new ArrayList<String>();
|
||||
visitedNodes.add(id);
|
||||
return flowTopologies.stream()
|
||||
// ignore already visited topologies
|
||||
.filter(x -> !visitedTopologies.contains(x.uid()))
|
||||
@@ -557,13 +551,8 @@ public class FlowService {
|
||||
visitedTopologies.add(topology.uid());
|
||||
Stream<FlowTopology> subTopologies = Stream
|
||||
.of(topology.getDestination(), topology.getSource())
|
||||
// ignore already visited nodes
|
||||
.filter(x -> !visitedNodes.contains(x.getId()))
|
||||
// recursively visit children and parents nodes
|
||||
.flatMap(relationNode -> {
|
||||
visitedNodes.add(relationNode.getId());
|
||||
return recursiveFlowTopology(visitedTopologies, relationNode.getTenantId(), relationNode.getNamespace(), relationNode.getId(), destinationOnly);
|
||||
});
|
||||
.flatMap(relationNode -> recursiveFlowTopology(visitedTopologies, relationNode.getTenantId(), relationNode.getNamespace(), relationNode.getId(), destinationOnly));
|
||||
return Stream.concat(Stream.of(topology), subTopologies);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
package io.kestra.core.services;
|
||||
|
||||
import io.kestra.core.repositories.KvMetadataRepositoryInterface;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.storages.kv.InternalKVStore;
|
||||
import io.kestra.core.storages.kv.KVStore;
|
||||
import io.kestra.core.storages.kv.KVStoreException;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Inject;
|
||||
import jakarta.inject.Singleton;
|
||||
@@ -14,8 +12,6 @@ import java.io.IOException;
|
||||
|
||||
@Singleton
|
||||
public class KVStoreService {
|
||||
@Inject
|
||||
private KvMetadataRepositoryInterface kvMetadataRepository;
|
||||
|
||||
@Inject
|
||||
private StorageInterface storageInterface;
|
||||
@@ -50,9 +46,9 @@ public class KVStoreService {
|
||||
boolean checkIfNamespaceExists = fromNamespace == null || isNotParentNamespace(namespace, fromNamespace);
|
||||
if (checkIfNamespaceExists && !namespaceService.isNamespaceExists(tenant, namespace)) {
|
||||
// if it didn't exist, we still check if there are KV as you can add KV without creating a namespace in DB or having flows in it
|
||||
KVStore kvStore = new InternalKVStore(tenant, namespace, storageInterface, kvMetadataRepository);
|
||||
KVStore kvStore = new InternalKVStore(tenant, namespace, storageInterface);
|
||||
try {
|
||||
if (kvStore.list(Pageable.from(1, 1)).isEmpty()) {
|
||||
if (kvStore.list().isEmpty()) {
|
||||
throw new KVStoreException(String.format(
|
||||
"Cannot access the KV store. The namespace '%s' does not exist.",
|
||||
namespace
|
||||
@@ -64,7 +60,7 @@ public class KVStoreService {
|
||||
return kvStore;
|
||||
}
|
||||
|
||||
return new InternalKVStore(tenant, namespace, storageInterface, kvMetadataRepository);
|
||||
return new InternalKVStore(tenant, namespace, storageInterface);
|
||||
}
|
||||
|
||||
private static boolean isNotParentNamespace(final String parentNamespace, final String childNamespace) {
|
||||
|
||||
@@ -58,10 +58,10 @@ import java.util.stream.Stream;
|
||||
public class PluginDefaultService {
|
||||
private static final ObjectMapper NON_DEFAULT_OBJECT_MAPPER = JacksonMapper.ofYaml()
|
||||
.copy()
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_DEFAULT);
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = JacksonMapper.ofYaml().copy()
|
||||
.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL);
|
||||
.setSerializationInclusion(JsonInclude.Include.NON_NULL);
|
||||
private static final String PLUGIN_DEFAULTS_FIELD = "pluginDefaults";
|
||||
|
||||
private static final TypeReference<List<PluginDefault>> PLUGIN_DEFAULTS_TYPE_REF = new TypeReference<>() {
|
||||
|
||||
@@ -5,14 +5,11 @@ import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import jakarta.annotation.Nullable;
|
||||
import jakarta.inject.Singleton;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Singleton
|
||||
@Slf4j
|
||||
public class SkipExecutionService {
|
||||
private volatile List<String> skipExecutions = Collections.emptyList();
|
||||
private volatile List<FlowId> skipFlows = Collections.emptyList();
|
||||
@@ -25,11 +22,11 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
public synchronized void setSkipFlows(List<String> skipFlows) {
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).filter(Objects::nonNull).toList();
|
||||
this.skipFlows = skipFlows == null ? Collections.emptyList() : skipFlows.stream().map(FlowId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipNamespaces(List<String> skipNamespaces) {
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).filter(Objects::nonNull).toList();
|
||||
this.skipNamespaces = skipNamespaces == null ? Collections.emptyList() : skipNamespaces.stream().map(NamespaceId::from).toList();
|
||||
}
|
||||
|
||||
public synchronized void setSkipTenants(List<String> skipTenants) {
|
||||
@@ -76,31 +73,19 @@ public class SkipExecutionService {
|
||||
}
|
||||
|
||||
record FlowId(String tenant, String namespace, String flow) {
|
||||
static @Nullable FlowId from(String flowId) {
|
||||
static FlowId from(String flowId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(flowId);
|
||||
if (parts.length == 3) {
|
||||
return new FlowId(parts[0], parts[1], parts[2]);
|
||||
} else if (parts.length == 2) {
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid flow skip with values: '{}'", flowId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new FlowId(null, parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
|
||||
record NamespaceId(String tenant, String namespace) {
|
||||
static @Nullable NamespaceId from(String namespaceId) {
|
||||
static NamespaceId from(String namespaceId) {
|
||||
String[] parts = SkipExecutionService.splitIdParts(namespaceId);
|
||||
|
||||
if (parts.length == 2) {
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
} else {
|
||||
log.error("Invalid namespace skip with values:'{}'", namespaceId);
|
||||
}
|
||||
|
||||
return null;
|
||||
return new NamespaceId(parts[0], parts[1]);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,12 +14,8 @@ import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.BiFunction;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwConsumer;
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
@@ -36,11 +32,7 @@ public abstract class StorageService {
|
||||
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(runContext.storage().getFile(from)))) {
|
||||
List<Path> splited;
|
||||
|
||||
if (storageSplitInterface.getRegexPattern() != null) {
|
||||
String renderedPattern = runContext.render(storageSplitInterface.getRegexPattern()).as(String.class).orElseThrow();
|
||||
String separator = runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow();
|
||||
splited = StorageService.splitByRegex(runContext, extension, separator, bufferedReader, renderedPattern);
|
||||
} else if (storageSplitInterface.getBytes() != null) {
|
||||
if (storageSplitInterface.getBytes() != null) {
|
||||
ReadableBytesTypeConverter readableBytesTypeConverter = new ReadableBytesTypeConverter();
|
||||
Number convert = readableBytesTypeConverter.convert(runContext.render(storageSplitInterface.getBytes()).as(String.class).orElseThrow(), Number.class)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Invalid size with value '" + storageSplitInterface.getBytes() + "'"));
|
||||
@@ -55,7 +47,7 @@ public abstract class StorageService {
|
||||
splited = StorageService.split(runContext, extension, runContext.render(storageSplitInterface.getSeparator()).as(String.class).orElseThrow(),
|
||||
bufferedReader, (bytes, size) -> size >= renderedRows);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, rows, nor regexPattern");
|
||||
throw new IllegalArgumentException("Invalid configuration with no size, count, nor rows");
|
||||
}
|
||||
|
||||
return splited
|
||||
@@ -125,36 +117,4 @@ public abstract class StorageService {
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
private static List<Path> splitByRegex(RunContext runContext, String extension, String separator, BufferedReader bufferedReader, String regexPattern) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
Map<String, RandomAccessFile> writers = new HashMap<>();
|
||||
Pattern pattern = Pattern.compile(regexPattern);
|
||||
|
||||
String row;
|
||||
while ((row = bufferedReader.readLine()) != null) {
|
||||
Matcher matcher = pattern.matcher(row);
|
||||
|
||||
if (matcher.find() && matcher.groupCount() > 0) {
|
||||
String routingKey = matcher.group(1);
|
||||
|
||||
// Get or create writer for this routing key
|
||||
RandomAccessFile writer = writers.get(routingKey);
|
||||
if (writer == null) {
|
||||
Path path = runContext.workingDir().createTempFile(extension);
|
||||
files.add(path);
|
||||
writer = new RandomAccessFile(path.toFile(), "rw");
|
||||
writers.put(routingKey, writer);
|
||||
}
|
||||
|
||||
byte[] bytes = (row + separator).getBytes(StandardCharsets.UTF_8);
|
||||
writer.getChannel().write(ByteBuffer.wrap(bytes));
|
||||
}
|
||||
// Lines that don't match the pattern are ignored
|
||||
}
|
||||
|
||||
writers.values().forEach(throwConsumer(RandomAccessFile::close));
|
||||
|
||||
return files.stream().filter(p -> p.toFile().length() > 0).toList();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -17,10 +17,6 @@ import java.net.URI;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* @deprecated use KVStore instead
|
||||
*/
|
||||
@Deprecated(since = "1.1.0", forRemoval = true)
|
||||
public record StateStore(RunContext runContext, boolean hashTaskRunValue) {
|
||||
|
||||
public InputStream getState(String stateName, @Nullable String stateSubName, String taskRunValue) throws IOException, ResourceExpiredException {
|
||||
|
||||
@@ -4,7 +4,6 @@ import com.google.common.annotations.VisibleForTesting;
|
||||
import io.kestra.core.models.executions.Execution;
|
||||
import io.kestra.core.models.executions.TaskRun;
|
||||
import io.kestra.core.models.flows.Flow;
|
||||
import io.kestra.core.models.flows.FlowId;
|
||||
import io.kestra.core.utils.Hashing;
|
||||
import io.kestra.core.utils.Slugify;
|
||||
import jakarta.annotation.Nullable;
|
||||
@@ -62,11 +61,11 @@ public class StorageContext {
|
||||
taskRun.getValue()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Factory method for constructing a new {@link StorageContext} scoped to a given {@link Flow}.
|
||||
*/
|
||||
public static StorageContext forFlow(FlowId flow) {
|
||||
public static StorageContext forFlow(Flow flow) {
|
||||
return new StorageContext(flow.getTenantId(), flow.getNamespace(), flow.getId());
|
||||
}
|
||||
|
||||
|
||||
@@ -360,41 +360,4 @@ public interface StorageInterface extends AutoCloseable, Plugin {
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the object name length does not exceed the allowed maximum.
|
||||
* If it does, the object name is truncated and a short random prefix is added
|
||||
* to avoid potential name collisions.
|
||||
*
|
||||
* @param uri the URI of the object
|
||||
* @param maxObjectNameLength the maximum allowed length for the object name
|
||||
* @return a normalized URI respecting the length limit
|
||||
* @throws IOException if the URI cannot be rebuilt
|
||||
*/
|
||||
default URI limit(URI uri, int maxObjectNameLength) throws IOException {
|
||||
if (uri == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String path = uri.getPath();
|
||||
String objectName = path.contains("/") ? path.substring(path.lastIndexOf("/") + 1) : path;
|
||||
|
||||
if (objectName.length() > maxObjectNameLength) {
|
||||
objectName = objectName.substring(objectName.length() - maxObjectNameLength + 6);
|
||||
String prefix = org.apache.commons.lang3.RandomStringUtils.secure()
|
||||
.nextAlphanumeric(5)
|
||||
.toLowerCase();
|
||||
|
||||
String newPath = (path.contains("/") ? path.substring(0, path.lastIndexOf("/") + 1) : "")
|
||||
+ prefix + "-" + objectName;
|
||||
|
||||
try {
|
||||
return new URI(uri.getScheme(), uri.getHost(), newPath, uri.getFragment());
|
||||
} catch (java.net.URISyntaxException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
return uri;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,13 +26,4 @@ public interface StorageSplitInterface {
|
||||
defaultValue = "\\n"
|
||||
)
|
||||
Property<String> getSeparator();
|
||||
|
||||
@Schema(
|
||||
title = "Split file by regex pattern with the first capture group as the routing key.",
|
||||
description = "A regular expression pattern with a capture group. Lines matching this pattern will be grouped by the captured value. " +
|
||||
"For example, `\\[(\\w+)\\]` will group lines by log level (ERROR, WARN, INFO) extracted from log entries. " +
|
||||
"Lines with the same captured value will be written to the same output file. " +
|
||||
"This allows content-based splitting where the file is divided based on data patterns rather than size or line count."
|
||||
)
|
||||
Property<String> getRegexPattern();
|
||||
}
|
||||
|
||||
@@ -1,32 +1,23 @@
|
||||
package io.kestra.core.storages.kv;
|
||||
|
||||
import io.kestra.core.exceptions.ResourceExpiredException;
|
||||
import io.kestra.core.models.FetchVersion;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||
import io.kestra.core.repositories.ArrayListTotal;
|
||||
import io.kestra.core.repositories.KvMetadataRepositoryInterface;
|
||||
import io.kestra.core.serializers.JacksonMapper;
|
||||
import io.kestra.core.storages.FileAttributes;
|
||||
import io.kestra.core.storages.StorageInterface;
|
||||
import io.kestra.core.storages.StorageObject;
|
||||
import io.kestra.core.utils.ListUtils;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
import io.micronaut.data.model.Sort;
|
||||
import jakarta.annotation.Nullable;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
|
||||
@@ -34,7 +25,6 @@ import static io.kestra.core.utils.Rethrow.throwFunction;
|
||||
* The default {@link KVStore} implementation.
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
public class InternalKVStore implements KVStore {
|
||||
|
||||
private static final Pattern DURATION_PATTERN = Pattern.compile("^P(?=[^T]|T.)(?:\\d*D)?(?:T(?=.)(?:\\d*H)?(?:\\d*M)?(?:\\d*S)?)?$");
|
||||
@@ -42,7 +32,6 @@ public class InternalKVStore implements KVStore {
|
||||
private final String namespace;
|
||||
private final String tenant;
|
||||
private final StorageInterface storage;
|
||||
private final KvMetadataRepositoryInterface kvMetadataRepository;
|
||||
|
||||
/**
|
||||
* Creates a new {@link InternalKVStore} instance.
|
||||
@@ -51,11 +40,10 @@ public class InternalKVStore implements KVStore {
|
||||
* @param tenant The tenant.
|
||||
* @param storage The storage.
|
||||
*/
|
||||
public InternalKVStore(@Nullable final String tenant, @Nullable final String namespace, final StorageInterface storage, final KvMetadataRepositoryInterface kvMetadataRepository) {
|
||||
this.namespace = namespace;
|
||||
public InternalKVStore(@Nullable final String tenant, final String namespace, final StorageInterface storage) {
|
||||
this.namespace = Objects.requireNonNull(namespace, "namespace cannot be null");
|
||||
this.storage = Objects.requireNonNull(storage, "storage cannot be null");
|
||||
this.tenant = tenant;
|
||||
this.kvMetadataRepository = kvMetadataRepository;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,18 +66,9 @@ public class InternalKVStore implements KVStore {
|
||||
"Cannot set value for key '%s'. Key already exists and `overwrite` is set to `false`.", key));
|
||||
}
|
||||
|
||||
Object actualValue = value.value();
|
||||
byte[] serialized = actualValue instanceof Duration ? actualValue.toString().getBytes(StandardCharsets.UTF_8) : JacksonMapper.ofIon().writeValueAsBytes(actualValue);
|
||||
byte[] serialized = JacksonMapper.ofIon().writeValueAsBytes(value.value());
|
||||
|
||||
PersistedKvMetadata saved = this.kvMetadataRepository.save(PersistedKvMetadata.builder()
|
||||
.tenantId(this.tenant)
|
||||
.namespace(this.namespace)
|
||||
.name(key)
|
||||
.description(Optional.ofNullable(value.metadata()).map(KVMetadata::getDescription).orElse(null))
|
||||
.expirationDate(Optional.ofNullable(value.metadata()).map(KVMetadata::getExpirationDate).orElse(null))
|
||||
.deleted(false)
|
||||
.build());
|
||||
this.storage.put(this.tenant, this.namespace, this.storageUri(key, saved.getVersion()), new StorageObject(
|
||||
this.storage.put(this.tenant, this.namespace, this.storageUri(key), new StorageObject(
|
||||
value.metadataAsMap(),
|
||||
new ByteArrayInputStream(serialized)
|
||||
));
|
||||
@@ -112,30 +91,20 @@ public class InternalKVStore implements KVStore {
|
||||
public Optional<String> getRawValue(String key) throws IOException, ResourceExpiredException {
|
||||
KVStore.validateKey(key);
|
||||
|
||||
Optional<PersistedKvMetadata> maybeMetadata = this.kvMetadataRepository.findByName(this.tenant, this.namespace, key);
|
||||
|
||||
int version = maybeMetadata.map(PersistedKvMetadata::getVersion).orElse(1);
|
||||
if (maybeMetadata.isPresent()) {
|
||||
PersistedKvMetadata metadata = maybeMetadata.get();
|
||||
if (metadata.isDeleted()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
if (Optional.ofNullable(metadata.getExpirationDate()).map(Instant.now()::isAfter).orElse(false)) {
|
||||
this.delete(key);
|
||||
throw new ResourceExpiredException("The requested value has expired");
|
||||
}
|
||||
}
|
||||
|
||||
StorageObject withMetadata;
|
||||
try {
|
||||
withMetadata = this.storage.getWithMetadata(this.tenant, this.namespace, this.storageUri(key, version));
|
||||
withMetadata = this.storage.getWithMetadata(this.tenant, this.namespace, this.storageUri(key));
|
||||
} catch (FileNotFoundException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
KVValueAndMetadata kvStoreValueWrapper = KVValueAndMetadata.from(withMetadata);
|
||||
|
||||
return Optional.of((String) (kvStoreValueWrapper.value()));
|
||||
Instant expirationDate = kvStoreValueWrapper.metadata().getExpirationDate();
|
||||
if (expirationDate != null && Instant.now().isAfter(expirationDate)) {
|
||||
this.delete(key);
|
||||
throw new ResourceExpiredException("The requested value has expired");
|
||||
}
|
||||
return Optional.of((String)(kvStoreValueWrapper.value()));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -144,13 +113,13 @@ public class InternalKVStore implements KVStore {
|
||||
@Override
|
||||
public boolean delete(String key) throws IOException {
|
||||
KVStore.validateKey(key);
|
||||
Optional<PersistedKvMetadata> maybeMetadata = this.kvMetadataRepository.findByName(this.tenant, this.namespace, key);
|
||||
if (maybeMetadata.map(PersistedKvMetadata::isDeleted).orElse(true)) {
|
||||
return false;
|
||||
URI uri = this.storageUri(key);
|
||||
boolean deleted = this.storage.delete(this.tenant, this.namespace, uri);
|
||||
URI metadataURI = URI.create(uri.getPath() + ".metadata");
|
||||
if (this.storage.exists(this.tenant, this.namespace, metadataURI)){
|
||||
this.storage.delete(this.tenant, this.namespace, metadataURI);
|
||||
}
|
||||
|
||||
this.kvMetadataRepository.delete(maybeMetadata.get());
|
||||
return true;
|
||||
return deleted;
|
||||
|
||||
}
|
||||
|
||||
@@ -158,27 +127,31 @@ public class InternalKVStore implements KVStore {
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public List<KVEntry> listAll() throws IOException {
|
||||
return this.list(Pageable.UNPAGED, Collections.emptyList(), true, true, FetchVersion.ALL);
|
||||
public List<KVEntry> list() throws IOException {
|
||||
List<FileAttributes> list = listAllFromStorage();
|
||||
return list.stream()
|
||||
.map(throwFunction(KVEntry::from))
|
||||
.filter(kvEntry -> Optional.ofNullable(kvEntry.expirationDate()).map(expirationDate -> Instant.now().isBefore(expirationDate)).orElse(true))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public ArrayListTotal<KVEntry> list(Pageable pageable, List<QueryFilter> filters, boolean allowDeleted, boolean allowExpired, FetchVersion fetchBehavior) throws IOException {
|
||||
if (this.namespace != null) {
|
||||
filters = Stream.concat(
|
||||
filters.stream(),
|
||||
Stream.of(QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value(this.namespace).build())
|
||||
).toList();
|
||||
}
|
||||
public List<KVEntry> listAll() throws IOException {
|
||||
List<FileAttributes> list = listAllFromStorage();
|
||||
return list.stream()
|
||||
.map(throwFunction(KVEntry::from))
|
||||
.toList();
|
||||
}
|
||||
|
||||
return this.kvMetadataRepository.find(
|
||||
pageable,
|
||||
this.tenant,
|
||||
filters,
|
||||
allowDeleted,
|
||||
allowExpired,
|
||||
fetchBehavior
|
||||
).map(throwFunction(KVEntry::from));
|
||||
private List<FileAttributes> listAllFromStorage() throws IOException {
|
||||
try {
|
||||
return this.storage.list(this.tenant, this.namespace, this.storageUri(null));
|
||||
} catch (FileNotFoundException e) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -188,39 +161,15 @@ public class InternalKVStore implements KVStore {
|
||||
public Optional<KVEntry> get(final String key) throws IOException {
|
||||
KVStore.validateKey(key);
|
||||
|
||||
Optional<PersistedKvMetadata> maybeMetadata = this.kvMetadataRepository.findByName(this.tenant, this.namespace, key);
|
||||
if (maybeMetadata.isEmpty() || maybeMetadata.get().isDeleted()) {
|
||||
try {
|
||||
KVEntry entry = KVEntry.from(this.storage.getAttributes(this.tenant, this.namespace, this.storageUri(key)));
|
||||
if (entry.expirationDate() != null && Instant.now().isAfter(entry.expirationDate())) {
|
||||
this.delete(key);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(entry);
|
||||
} catch (FileNotFoundException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(KVEntry.from(maybeMetadata.get()));
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public Integer purge(List<KVEntry> kvEntries) throws IOException {
|
||||
Integer purgedMetadataCount = this.kvMetadataRepository.purge(kvEntries.stream().map(kv -> PersistedKvMetadata.from(tenant, kv)).toList());
|
||||
|
||||
long actualDeletedEntries = kvEntries.stream()
|
||||
.map(KVEntry::key)
|
||||
.map(this::storageUri)
|
||||
.map(throwFunction(uri -> {
|
||||
boolean deleted = this.storage.delete(tenant, namespace, uri);
|
||||
URI metadataURI = URI.create(uri.getPath() + ".metadata");
|
||||
if (this.storage.exists(this.tenant, this.namespace, metadataURI)) {
|
||||
this.storage.delete(this.tenant, this.namespace, metadataURI);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
})).filter(Boolean::booleanValue)
|
||||
.count();
|
||||
|
||||
if (actualDeletedEntries != purgedMetadataCount) {
|
||||
log.warn("KV Metadata purge reported {} deleted entries, but {} values were actually deleted from storage", purgedMetadataCount, actualDeletedEntries);
|
||||
}
|
||||
|
||||
return purgedMetadataCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
package io.kestra.core.storages.kv;
|
||||
|
||||
import io.kestra.core.models.kv.PersistedKvMetadata;
|
||||
import io.kestra.core.storages.FileAttributes;
|
||||
import jakarta.annotation.Nullable;
|
||||
|
||||
@@ -8,23 +7,12 @@ import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public record KVEntry(String namespace, String key, Integer version, @Nullable String description, Instant creationDate, Instant updateDate, @Nullable Instant expirationDate) {
|
||||
private static final Pattern captureKeyAndVersion = Pattern.compile("(.*)\\.ion(?:\\.v(\\d+))?$");
|
||||
|
||||
public static KVEntry from(String namespace, FileAttributes fileAttributes) throws IOException {
|
||||
public record KVEntry(String key, @Nullable String description, Instant creationDate, Instant updateDate, @Nullable Instant expirationDate) {
|
||||
public static KVEntry from(FileAttributes fileAttributes) throws IOException {
|
||||
Optional<KVMetadata> kvMetadata = Optional.ofNullable(fileAttributes.getMetadata()).map(KVMetadata::new);
|
||||
String fileName = fileAttributes.getFileName();
|
||||
Matcher matcher = captureKeyAndVersion.matcher(fileName);
|
||||
if (!matcher.matches()) {
|
||||
throw new IOException("Invalid KV file name format: " + fileName);
|
||||
}
|
||||
return new KVEntry(
|
||||
namespace,
|
||||
matcher.group(1),
|
||||
Optional.ofNullable(matcher.group(2)).map(Integer::parseInt).orElse(1),
|
||||
fileAttributes.getFileName().replace(".ion", ""),
|
||||
kvMetadata.map(KVMetadata::getDescription).orElse(null),
|
||||
Instant.ofEpochMilli(fileAttributes.getCreationTime()),
|
||||
Instant.ofEpochMilli(fileAttributes.getLastModifiedTime()),
|
||||
@@ -33,8 +21,4 @@ public record KVEntry(String namespace, String key, Integer version, @Nullable S
|
||||
.orElse(null)
|
||||
);
|
||||
}
|
||||
|
||||
public static KVEntry from(PersistedKvMetadata persistedKvMetadata) throws IOException {
|
||||
return new KVEntry(persistedKvMetadata.getNamespace(), persistedKvMetadata.getName(), persistedKvMetadata.getVersion(), persistedKvMetadata.getDescription(), persistedKvMetadata.getCreated(), persistedKvMetadata.getUpdated(), persistedKvMetadata.getExpirationDate());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
package io.kestra.core.storages.kv;
|
||||
|
||||
import io.kestra.core.exceptions.ResourceExpiredException;
|
||||
import io.kestra.core.models.FetchVersion;
|
||||
import io.kestra.core.models.QueryFilter;
|
||||
import io.kestra.core.repositories.ArrayListTotal;
|
||||
import io.kestra.core.storages.StorageContext;
|
||||
import io.micronaut.data.model.Pageable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.net.URI;
|
||||
import java.util.Collections;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -29,15 +25,11 @@ public interface KVStore {
|
||||
String namespace();
|
||||
|
||||
default URI storageUri(String key) {
|
||||
return this.storageUri(key, 1);
|
||||
return this.storageUri(key, namespace());
|
||||
}
|
||||
|
||||
default URI storageUri(String key, int version) {
|
||||
return this.storageUri(key, namespace(), version);
|
||||
}
|
||||
|
||||
default URI storageUri(String key, String namespace, int version) {
|
||||
String filePath = key == null ? "" : ("/" + key + ".ion") + (version > 1 ? (".v" + version) : "");
|
||||
default URI storageUri(String key, String namespace) {
|
||||
String filePath = key == null ? "" : ("/" + key + ".ion");
|
||||
return URI.create(StorageContext.KESTRA_PROTOCOL + StorageContext.kvPrefix(namespace) + filePath);
|
||||
}
|
||||
|
||||
@@ -80,30 +72,13 @@ public interface KVStore {
|
||||
*/
|
||||
boolean delete(String key) throws IOException;
|
||||
|
||||
/**
|
||||
* Purge the provided KV entries.
|
||||
*/
|
||||
Integer purge(List<KVEntry> kvToDelete) throws IOException;
|
||||
|
||||
default ArrayListTotal<KVEntry> list() throws IOException {
|
||||
return this.list(Pageable.UNPAGED);
|
||||
}
|
||||
|
||||
default ArrayListTotal<KVEntry> list(Pageable pageable) throws IOException {
|
||||
return this.list(pageable, Collections.emptyList());
|
||||
}
|
||||
|
||||
default ArrayListTotal<KVEntry> list(Pageable pageable, List<QueryFilter> queryFilters) throws IOException {
|
||||
return this.list(pageable, queryFilters, false, false, FetchVersion.LATEST);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all the K/V store entries.
|
||||
*
|
||||
* @return The list of {@link KVEntry}.
|
||||
* @throws IOException if an error occurred while executing the operation on the K/V store.
|
||||
*/
|
||||
ArrayListTotal<KVEntry> list(Pageable pageable, List<QueryFilter> queryFilters, boolean allowDeleted, boolean allowExpired, FetchVersion fetchBehavior) throws IOException;
|
||||
List<KVEntry> list() throws IOException;
|
||||
|
||||
/**
|
||||
* Lists all the K/V store entries, expired or not.
|
||||
@@ -122,22 +97,22 @@ public interface KVStore {
|
||||
Optional<KVEntry> get(String key) throws IOException;
|
||||
|
||||
/**
|
||||
* Checks whether a K/V entry exists for the given key.
|
||||
* Checks whether a K/V entry exists for teh given key.
|
||||
*
|
||||
* @param key The entry key.
|
||||
* @return {@code true} of an entry exists.
|
||||
* @throws IOException if an error occurred while executing the operation on the K/V store.
|
||||
*/
|
||||
default boolean exists(String key) throws IOException {
|
||||
return get(key).isPresent();
|
||||
return list().stream().anyMatch(kvEntry -> kvEntry.key().equals(key));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Finds a KV entry with associated metadata for a given key.
|
||||
*
|
||||
* @param key the KV entry key.
|
||||
* @return an optional of {@link KVValueAndMetadata}.
|
||||
*
|
||||
*
|
||||
* @throws UncheckedIOException if an error occurred while executing the operation on the K/V store.
|
||||
*/
|
||||
default Optional<KVValueAndMetadata> findMetadataAndValue(final String key) throws UncheckedIOException {
|
||||
@@ -157,7 +132,7 @@ public interface KVStore {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Pattern KEY_VALIDATOR_PATTERN = Pattern.compile("[a-zA-Z0-9][a-zA-Z0-9._-]*");
|
||||
|
||||
/**
|
||||
|
||||
@@ -70,7 +70,7 @@ public class FlowTopologyService {
|
||||
}
|
||||
|
||||
public FlowTopologyGraph namespaceGraph(String tenantId, String namespace) {
|
||||
List<FlowTopology> flowTopologies = flowTopologyRepository.findByNamespacePrefix(tenantId, namespace);
|
||||
List<FlowTopology> flowTopologies = flowTopologyRepository.findByNamespace(tenantId, namespace);
|
||||
|
||||
FlowTopologyGraph graph = this.graph(flowTopologies.stream(), (flowNode -> flowNode));
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user