Compare commits

...

203 Commits

Author SHA1 Message Date
Sandip Mandal
f5a0dcc024 chore(core): make sure kv listing is filterable (#11536)
Closes https://github.com/kestra-io/kestra/issues/11413.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-29 09:30:09 +02:00
Satvik Kushwaha
5c079b8b6b chore(namespaces): update page title on single namespace page (#11551)
Closes https://github.com/kestra-io/kestra/issues/11428.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-29 09:21:26 +02:00
Barthélémy Ledoux
343d6b4eb9 refactor(plugins): update documentation to use typescript and composition api (#11543) 2025-09-27 09:33:26 +01:00
Kenneth Rabe
d34d547412 fix(pebble): correct return format of timestampMicro 2025-09-26 16:51:35 +02:00
Nicolas K.
7a542a24e2 fix(executor): remove debug log (#11548)
Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-26 15:03:08 +02:00
Nicolas K.
5b1db68752 fix(test): flaky test with unwanted repeat test annotation (#11547)
Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-26 14:50:26 +02:00
Nicolas K.
5b07b643d3 fix(test): disable flaky test and add configuration to the ELS indexe… (#11539)
* fix(test): disable flaky test and add configuration to the ELS indexer poll duration

* fix(test): retry a flaky test and fix a flaky

* feat(test): disable a test until we have time to fix the bug

---------

Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-26 14:19:20 +02:00
Barthélémy Ledoux
0e059772e4 chore: remove posthog in dev mode (#11540) 2025-09-26 10:49:27 +01:00
Loïc Mathieu
f72e294e54 chore(system): log machine information at startup
This will log this kind of line at startup, helping to understand possible infrastructure limitation by looking at the starting logs.

```
14:38:17.018 INFO  main         i.k.c.c.s.AbstractServerCommand Machine information: 16 available cpu(s), 2048MB max memory, Java version 21.0.5+11-LTS
```
2025-09-26 10:55:05 +02:00
Loïc Mathieu
98dd884149 chore(executions): always log errors from the executor
- Logs errors from the Executor catched execution
- Logs errors from the Scheduler catched execution
- Avoid most places where the warning "unable to change state already..." could occur
- Log using the run context logger flow issues from executable tasks so they appears inside execution logs
2025-09-26 10:43:05 +02:00
Loïc Mathieu
26c4f080fd chore(deps): use the version of bcpkix-jdk18on from the platform 2025-09-26 10:42:47 +02:00
yuri1969
01293de91c fix(core): enable runIf at execution updating tasks 2025-09-25 10:23:13 +02:00
Mustafa Tarek
892b69f10e fix(core): Add warning logs for mismatched (Parent-Subflow) inputs (#11431)
* fix(core): Add warning logs for mismatched (Parent-Subflow) inputs for subflow plugin.

* feat: add check and log to FlowInputOutput.java

* enhancement: avoid unnecessary input validation in ExecutableUtils.subflowExecution() when no mismatches exist
2025-09-25 10:08:37 +02:00
yuri1969
6f70d4d275 fix(core): amend test
Adjusted to e1d2c30e which made the execution fail on empty value.
2025-09-25 09:49:19 +02:00
yuri1969
b41d2e456f fix(core): do not allow empty labels
* Filtered empty  entries on Labels task.
* Checking empty Flow labels via validation.
* Adjusted UI to disallow setting empty labels.
2025-09-25 09:49:19 +02:00
UncleBigBay
5ec08eda8c feat (layout): new sidebar total collapse behaviour (#11471)
Co-authored-by: Piyush Bhaskar <102078527+Piyush-r-bhaskar@users.noreply.github.com>
Co-authored-by: Piyush Bhaskar <impiyush0012@gmail.com>
2025-09-25 12:06:24 +05:30
dependabot[bot]
7ed6b883ff build(deps): bump io.micronaut.openapi:micronaut-openapi-bom
Bumps [io.micronaut.openapi:micronaut-openapi-bom](https://github.com/micronaut-projects/micronaut-openapi) from 6.18.0 to 6.18.1.
- [Release notes](https://github.com/micronaut-projects/micronaut-openapi/releases)
- [Commits](https://github.com/micronaut-projects/micronaut-openapi/compare/v6.18.0...v6.18.1)

---
updated-dependencies:
- dependency-name: io.micronaut.openapi:micronaut-openapi-bom
  dependency-version: 6.18.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:22:28 +02:00
dependabot[bot]
eb166c9321 build(deps): bump jakarta.mail:jakarta.mail-api from 2.1.4 to 2.1.5
Bumps [jakarta.mail:jakarta.mail-api](https://github.com/jakartaee/mail-api) from 2.1.4 to 2.1.5.
- [Release notes](https://github.com/jakartaee/mail-api/releases)
- [Commits](https://github.com/jakartaee/mail-api/compare/2.1.4...2.1.5)

---
updated-dependencies:
- dependency-name: jakarta.mail:jakarta.mail-api
  dependency-version: 2.1.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:22:05 +02:00
dependabot[bot]
57aad1b931 build(deps): bump software.amazon.awssdk.crt:aws-crt
Bumps [software.amazon.awssdk.crt:aws-crt](https://github.com/awslabs/aws-crt-java) from 0.38.13 to 0.39.0.
- [Release notes](https://github.com/awslabs/aws-crt-java/releases)
- [Commits](https://github.com/awslabs/aws-crt-java/compare/v0.38.13...v0.39.0)

---
updated-dependencies:
- dependency-name: software.amazon.awssdk.crt:aws-crt
  dependency-version: 0.39.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:21:47 +02:00
dependabot[bot]
60fe5b5c76 build(deps): bump org.apache.logging.log4j:log4j-to-slf4j
Bumps org.apache.logging.log4j:log4j-to-slf4j from 2.25.1 to 2.25.2.

---
updated-dependencies:
- dependency-name: org.apache.logging.log4j:log4j-to-slf4j
  dependency-version: 2.25.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:21:29 +02:00
dependabot[bot]
98c69b53bb build(deps): bump software.amazon.awssdk:bom from 2.33.11 to 2.34.2
Bumps software.amazon.awssdk:bom from 2.33.11 to 2.34.2.

---
updated-dependencies:
- dependency-name: software.amazon.awssdk:bom
  dependency-version: 2.34.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:21:10 +02:00
dependabot[bot]
d5d38559b4 build(deps): bump com.github.oshi:oshi-core from 6.8.3 to 6.9.0
Bumps [com.github.oshi:oshi-core](https://github.com/oshi/oshi) from 6.8.3 to 6.9.0.
- [Release notes](https://github.com/oshi/oshi/releases)
- [Changelog](https://github.com/oshi/oshi/blob/master/CHANGELOG.md)
- [Commits](https://github.com/oshi/oshi/compare/oshi-parent-6.8.3...oshi-parent-6.9.0)

---
updated-dependencies:
- dependency-name: com.github.oshi:oshi-core
  dependency-version: 6.9.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:20:50 +02:00
dependabot[bot]
4273ddc4f6 build(deps): bump org.apache.httpcomponents.core5:httpcore5-h2
Bumps [org.apache.httpcomponents.core5:httpcore5-h2](https://github.com/apache/httpcomponents-core) from 5.3.5 to 5.3.6.
- [Changelog](https://github.com/apache/httpcomponents-core/blob/rel/v5.3.6/RELEASE_NOTES.txt)
- [Commits](https://github.com/apache/httpcomponents-core/compare/rel/v5.3.5...rel/v5.3.6)

---
updated-dependencies:
- dependency-name: org.apache.httpcomponents.core5:httpcore5-h2
  dependency-version: 5.3.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:20:10 +02:00
dependabot[bot]
980c573a30 build(deps): bump org.eclipse.angus:jakarta.mail from 2.0.4 to 2.0.5
Bumps org.eclipse.angus:jakarta.mail from 2.0.4 to 2.0.5.

---
updated-dependencies:
- dependency-name: org.eclipse.angus:jakarta.mail
  dependency-version: 2.0.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:18:43 +02:00
dependabot[bot]
27109015f9 build(deps): bump org.projectlombok:lombok from 1.18.40 to 1.18.42
Bumps [org.projectlombok:lombok](https://github.com/projectlombok/lombok) from 1.18.40 to 1.18.42.
- [Changelog](https://github.com/projectlombok/lombok/blob/master/doc/changelog.markdown)
- [Commits](https://github.com/projectlombok/lombok/compare/v1.18.40...v1.18.42)

---
updated-dependencies:
- dependency-name: org.projectlombok:lombok
  dependency-version: 1.18.42
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:18:15 +02:00
dependabot[bot]
eba7d4f375 build(deps): bump bouncycastleVersion from 1.81 to 1.82
Bumps `bouncycastleVersion` from 1.81 to 1.82.

Updates `org.bouncycastle:bcprov-jdk18on` from 1.81 to 1.82
- [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html)
- [Commits](https://github.com/bcgit/bc-java/commits)

Updates `org.bouncycastle:bcpg-jdk18on` from 1.81 to 1.82
- [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html)
- [Commits](https://github.com/bcgit/bc-java/commits)

Updates `org.bouncycastle:bcpkix-jdk18on` from 1.81 to 1.82
- [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html)
- [Commits](https://github.com/bcgit/bc-java/commits)

---
updated-dependencies:
- dependency-name: org.bouncycastle:bcprov-jdk18on
  dependency-version: '1.82'
  dependency-type: direct:production
  update-type: version-update:semver-minor
- dependency-name: org.bouncycastle:bcpg-jdk18on
  dependency-version: '1.82'
  dependency-type: direct:production
  update-type: version-update:semver-minor
- dependency-name: org.bouncycastle:bcpkix-jdk18on
  dependency-version: '1.82'
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 16:17:53 +02:00
dependabot[bot]
655a1172ee build(deps): bump org.assertj:assertj-core from 3.27.4 to 3.27.6
Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.27.4 to 3.27.6.
- [Release notes](https://github.com/assertj/assertj/releases)
- [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.27.4...assertj-build-3.27.6)

---
updated-dependencies:
- dependency-name: org.assertj:assertj-core
  dependency-version: 3.27.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 15:45:31 +02:00
dependabot[bot]
6e49a85acd build(deps): bump org.owasp.dependencycheck from 12.1.3 to 12.1.5
Bumps org.owasp.dependencycheck from 12.1.3 to 12.1.5.

---
updated-dependencies:
- dependency-name: org.owasp.dependencycheck
  dependency-version: 12.1.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-24 15:44:40 +02:00
Barthélémy Ledoux
4515bad6bd fix(flows): delete flows should work (#11469) 2025-09-24 09:35:47 +02:00
Loïc Mathieu
226dbd30c9 fix(tests): fix test flow namespace and id 2025-09-24 09:19:31 +02:00
mustafatarek
6b0c190edc feat: added test case covering ForEach Iteration 2025-09-24 09:19:31 +02:00
mustafatarek
c64df40a36 refactor: change iteration to start with 0 2025-09-24 09:19:31 +02:00
mustafatarek
8af22d1bb2 fix(core): fix ForEach plugin task.iteration property to show the correct number of Iteration 2025-09-24 09:19:31 +02:00
Nicolas K.
b294457953 feat(tests): rework runner utils to not use the queue during testing (#11380)
* feat(tests): rework runner utils to not use the queue during testing

* feat(tests): rework runner utils to not use the queue during testing

* test: rework RetryCaseTest to not rely on executionQueue

* fix(tests): don't catch the Queue exception

* fix(tests): don't catch the Queue exception

* fix compile

* fix(test): concurrency error and made runner test parallel ready

* fix(tests): remove test instance

* feat(tests): use Test Runner Utils

* fix(tests): flaky tests

* fix(test): flaky tests

* feat(tests): rework runner utils to not use the queue during testing

* feat(tests): rework runner utils to not use the queue during testing

* test: rework RetryCaseTest to not rely on executionQueue

* fix(tests): don't catch the Queue exception

* fix(tests): don't catch the Queue exception

* fix compile

* fix(test): concurrency error and made runner test parallel ready

* fix(tests): remove test instance

* feat(tests): use Test Runner Utils

* fix(tests): flaky tests

* fix(test): flaky tests

* fix(tests): flaky set test

* fix(tests): remove RunnerUtils

* fix(tests): fix flaky

* feat(test): rework runner tests to remove the queue usage

* feat(test): fix a flaky and remove parallelism from mysql test suit

* fix(tests): flaky tests

* clean(tests): unwanted test

* add debug exec when fail

* feat(tests): add thread to mysql thread pool

* fix(test): flaky and disable a test

---------

Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
Co-authored-by: Roman Acevedo <roman.acevedo62@gmail.com>
2025-09-24 08:18:02 +02:00
Loïc Mathieu
02d9c589fb chore(system): remove the task run page
Part-of: https://github.com/kestra-io/kestra-ee/issues/5174
2025-09-23 14:48:30 +02:00
Sanjay Ramsinghani
6340d1c72f chore(core): align toggle icon in failed execution collapse element (#11430)
Closes https://github.com/kestra-io/kestra/issues/11406.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-23 14:14:24 +02:00
Raj Gupta
f439bd53d7 chore(system): clean up filters config (#11405)
closes #11378
2025-09-23 14:11:00 +02:00
github-actions[bot]
e54e3d5308 chore(core): localize to languages other than english (#11464)
Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference.

Co-authored-by: GitHub Action <actions@github.com>
2025-09-23 13:43:37 +02:00
Emmanuel Adeniyi Adekeye
d084f2cd26 chore(core): improve display format in dashboard charts (#11456)
Closes https://github.com/kestra-io/kestra/issues/11171.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-23 13:41:55 +02:00
Ritoban Dutta
015960c78e chore(core): update background color of context panel menu (#11441)
Closes https://github.com/kestra-io/kestra/issues/11426.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-23 12:57:12 +02:00
yummyash
39a09ecb67 chore(core): replace illustration on multi-panel empty screen (#11457)
Closes https://github.com/kestra-io/kestra/issues/11244.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-23 12:54:19 +02:00
Miloš Paunović
45ce878d65 fix(executions): properly parse defaults for json-type inputs (#11459)
Closes https://github.com/kestra-io/kestra/issues/11449.
2025-09-23 12:48:05 +02:00
Miloš Paunović
3ee647b9a8 feat(triggers): show dialog when clicking the backfill button (#11445)
Closes https://github.com/kestra-io/kestra/issues/11433.
2025-09-23 12:24:57 +02:00
github-actions[bot]
7a7cb006bf chore(core): localize to languages other than english (#11455)
Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference.

Co-authored-by: GitHub Action <actions@github.com>
2025-09-23 11:06:08 +02:00
Piyush Bhaskar
911e6d5705 fix(core): fix pagination load data changes (#11453) 2025-09-23 14:25:20 +05:30
brian-mulier-p
bf1458dde7 fix(ai): avoid moving cursor twice after using AI Copilot (#11451)
closes #11314
2025-09-23 10:30:37 +02:00
yuri1969
bd31e0eebd fix(tests): sanitize E2E control scripts 2025-09-23 10:16:06 +02:00
Florian Hussonnois
de02e4dd70 fix(triggers): handle RecoverMissedSchedules on trigger batch update
* Fix and clean code in TriggerController
* Remove duplicate code in Trigger class
2025-09-23 10:06:03 +02:00
Karuna Tata
ec235b91fc feat: add fix with AI button for error tasks (#11416) 2025-09-23 11:53:07 +05:30
brian.mulier
ff1efa9958 fix(system): avoid trigger locking after scheduler restart
closes #11434
2025-09-22 19:26:58 +02:00
brian.mulier
e43c8ce387 fix(ci): add on pull request for release branches 2025-09-22 19:05:05 +02:00
Barthélémy Ledoux
2bd4e82b42 chore: make a lot of progress on the typescript front (#11234)
Co-authored-by: Piyush Bhaskar <impiyush0012@gmail.com>
2025-09-22 13:39:52 +02:00
Jeffrey Ricker
e63d6d1d86 feat(flows): add Pebble nanoId function
* nanoid function

* nanoid function

* Update core/src/main/java/io/kestra/core/runners/pebble/functions/NanoIDFunction.java

Co-authored-by: Roman Acevedo <roman.acevedo62@gmail.com>

* Update core/src/test/java/io/kestra/core/runners/pebble/functions/NanoIDFuntionTest.java

Co-authored-by: Roman Acevedo <roman.acevedo62@gmail.com>

* fix: nanoId to parse Long to work with Kestra yaml

---------

Co-authored-by: Roman Acevedo <roman.acevedo62@gmail.com>
2025-09-22 11:30:44 +02:00
Florian Hussonnois
a9752e65f2 chore(websever): make kvStore method in KVController protected
Related-to: kestra-io/kestra-ee#5055
2025-09-22 11:30:10 +02:00
Florian Hussonnois
151c56f3de feat(core): add new findMetadataAndValue to KVStore
Related-to: kestra-io/kestra-ee#5055
2025-09-22 11:30:10 +02:00
Piyush Bhaskar
d562ce3e65 fix: fix ci build error 2025-09-22 11:06:59 +02:00
Florian Hussonnois
681386a05b fix(kvstores): fix description not set when editing 2025-09-22 11:06:59 +02:00
Sehnya
51ddfaf155 chore(core): remove shadow above the user selector (#11364)
Closes https://github.com/kestra-io/kestra/issues/11352.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-22 09:26:24 +02:00
Pratyush Kumar
caee0a293f Add feature to also show an Outputs column in the summary table (#11412)
Co-authored-by: Piyush Bhaskar <102078527+Piyush-r-bhaskar@users.noreply.github.com>
2025-09-22 12:25:21 +05:30
github-actions[bot]
ba92880fa3 chore(core): localize to languages other than english (#11414)
Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference.

Co-authored-by: GitHub Action <actions@github.com>
2025-09-22 08:48:58 +02:00
rhodemilk
36b27510fb chore(executions): rename label in overview page (#11363)
Closes https://github.com/kestra-io/kestra/issues/11287.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-22 08:46:48 +02:00
Siddharthrane07
da2907e096 fix(core): show proper light mode tooltip. (#11407)
Co-authored-by: Piyush Bhaskar <impiyush0012@gmail.com>
2025-09-22 12:01:12 +05:30
Ludovic DEHON
9b40665e64 chore(build): use remote actions for java report 2025-09-19 23:56:32 +02:00
Ludovic DEHON
0d35b5b355 fix(system): make skip flow and namespace resilient to errors 2025-09-19 23:53:17 +02:00
Roman Acevedo
339eb79854 ci: migrate kestra-devtools to npm 2025-09-19 16:59:59 +02:00
brian-mulier-p
0ee753529b fix(tests): enforce closing consumers after each tests (#11399) 2025-09-19 16:27:37 +02:00
Miloš Paunović
84668fdfb9 chore(namespaces): add disabled attribute to typescript interface for namespace tabs (#11398)
Related to https://github.com/kestra-io/kestra-ee/issues/4726.
2025-09-19 13:54:37 +02:00
Miloš Paunović
9802f046e8 chore(triggers): amend labels on table column header and switch element (#11395)
Closes https://github.com/kestra-io/kestra-ee/issues/5196.
2025-09-19 11:40:38 +02:00
brian-mulier-p
848b4d6577 fix(core): avoid ClassCastException when doing secret decryption (#11393)
closes kestra-io/kestra-ee#5191
2025-09-19 11:24:39 +02:00
github-actions[bot]
1159bc5eb9 chore(core): localize to languages other than english (#11394)
Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference.

Co-authored-by: GitHub Action <actions@github.com>
2025-09-19 11:05:14 +02:00
Sandip Mandal
9b7ef37d14 fix(core: webhook curl coomand needs tenant. (#11391)
Co-authored-by: Piyush Bhaskar <102078527+Piyush-r-bhaskar@users.noreply.github.com>
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-19 14:06:08 +05:30
Piyush Bhaskar
89dfd18658 fix(admin): humanize label for trigger keys (#11377)
Co-authored-by: GitHub Action <actions@github.com>
Co-authored-by: MilosPaunovic <paun992@hotmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2025-09-19 12:08:20 +05:30
Loïc Mathieu
13ed2252bc fix(executions): computing subflow outputs could fail when the executioin is failing or killing
Fixes https://github.com/kestra-io/kestra/issues/11379
2025-09-18 17:41:13 +02:00
Barthélémy Ledoux
c73b103bb3 fix: restore sidebar icon zindex (#11371) 2025-09-18 14:36:45 +02:00
Will Russell
396a077942 docs(overview): new video (#11374) 2025-09-18 12:56:30 +01:00
Miloš Paunović
68e6fa2a4c feat(flows): save editor panel layout after creation (#11276)
Closes https://github.com/kestra-io/kestra/issues/9887.

Co-authored-by: Bart Ledoux <bledoux@kestra.io>
2025-09-18 12:47:31 +02:00
Piyush Bhaskar
a18748b3b2 refactor(core): remove TaskRuns page (#11366) 2025-09-18 15:58:20 +05:30
Loïc Mathieu
236fcff7b4 fix(executions): concurrency limit should update the executioin
As if it's not updated in the database, it would not be detected as changed so that terminal actions (like purge) would not be done.

Fixes  #11022
Fixes #11025
Fixes #8143
2025-09-18 12:08:54 +02:00
Roman Acevedo
cbbd697732 ci: hide by default error logs in test report PR (#11354) 2025-09-18 12:04:30 +02:00
Loïc Mathieu
6b84737651 fix(executions): the Exit task was not correctly ends parent tasks
Fixes https://github.com/kestra-io/kestra-ee/issues/5168
2025-09-18 11:36:23 +02:00
Florian Hussonnois
6ee7ecbd6b fix(core): fix NPE in JackMapping.applyPatchesOnJsonNode method 2025-09-18 10:57:00 +02:00
Roman Acevedo
504f925085 test: make AbstractExecutionRepositoryTest parallelizable (#11295)
* test: make AbstractExecutionRepositoryTest parallelizable

* feat(tests): play jdbc h2 tests in parallel

* fix(tests): failing unit tests

* tests: add await until timeout on some tests

* fix(tests): failing unit tests

* fix(tests): failing unit tests

---------

Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
Co-authored-by: Nicolas K. <nk_mikmak@hotmail.com>
2025-09-17 17:41:10 +02:00
Loïc Mathieu
7d37d2be93 fix(executions): possible NPE on dynamic taskrun
Fixes https://github.com/kestra-io/kestra-ee/issues/5166
2025-09-17 15:55:52 +02:00
brian.mulier
94751a3b21 fix(core): filters weren't applying anymore 2025-09-17 12:55:17 +02:00
Piyush Bhaskar
ba83b91680 fix(core): conditionally disable current route (#11353) 2025-09-17 15:51:12 +05:30
Piyush Bhaskar
56f62fb89f fix(core): show subgroups card only if exist, else show main group (#11348)
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-17 15:18:21 +05:30
Loïc Mathieu
a0efe4b1f3 fix(system): properly pass Micronaut env 2025-09-17 11:35:59 +02:00
dependabot[bot]
9af6338ae5 build(deps): bump dev.langchain4j:langchain4j-bom from 1.4.0 to 1.5.0
Bumps [dev.langchain4j:langchain4j-bom](https://github.com/langchain4j/langchain4j) from 1.4.0 to 1.5.0.
- [Release notes](https://github.com/langchain4j/langchain4j/releases)
- [Commits](https://github.com/langchain4j/langchain4j/compare/1.4.0...1.5.0)

---
updated-dependencies:
- dependency-name: dev.langchain4j:langchain4j-bom
  dependency-version: 1.5.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 11:32:55 +02:00
dependabot[bot]
d53b933bdf build(deps): bump jakarta.xml.bind:jakarta.xml.bind-api
Bumps [jakarta.xml.bind:jakarta.xml.bind-api](https://github.com/jakartaee/jaxb-api) from 4.0.2 to 4.0.4.
- [Release notes](https://github.com/jakartaee/jaxb-api/releases)
- [Commits](https://github.com/jakartaee/jaxb-api/compare/4.0.2...4.0.4)

---
updated-dependencies:
- dependency-name: jakarta.xml.bind:jakarta.xml.bind-api
  dependency-version: 4.0.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 11:32:43 +02:00
dependabot[bot]
a35c2816c5 build(deps): bump dev.langchain4j:langchain4j-community-bom
Bumps [dev.langchain4j:langchain4j-community-bom](https://github.com/langchain4j/langchain4j-community) from 1.4.0-beta10 to 1.5.0-beta11.
- [Release notes](https://github.com/langchain4j/langchain4j-community/releases)
- [Commits](https://github.com/langchain4j/langchain4j-community/compare/1.4.0-beta10...1.5.0-beta11)

---
updated-dependencies:
- dependency-name: dev.langchain4j:langchain4j-community-bom
  dependency-version: 1.5.0-beta11
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 11:31:47 +02:00
Loïc Mathieu
d88eb9974c fix(system): use the Langchain Gemini lib version from the platform 2025-09-17 11:14:11 +02:00
Florian Hussonnois
af3d5a384a fix(core): fix plugin stable version resolution (kestra-io/kestra-ee#5129)
Rename incremental field to patch

Fixes: kestra-io/kestra-ee#5129
2025-09-17 11:10:40 +02:00
Miloš Paunović
e9ad352ccf chore(core): remove unused decompress library (#11346) 2025-09-17 11:04:36 +02:00
yuri
1a95b83fb7 chore(logs): make search queries case-insensitive (#11313)
Execution logs' filter query used to be case-sensitive - for example, the `hello` query did not match `Hello World` log lines.
2025-09-17 11:04:07 +02:00
dependabot[bot]
095939ff7a build(deps): bump software.amazon.awssdk:bom from 2.33.5 to 2.33.11
Bumps software.amazon.awssdk:bom from 2.33.5 to 2.33.11.

---
updated-dependencies:
- dependency-name: software.amazon.awssdk:bom
  dependency-version: 2.33.11
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:46:42 +02:00
Piyush Bhaskar
094f523874 fix(core): remove object Object from tab title. (#11347) 2025-09-17 14:08:50 +05:30
brian.mulier
c7efb2514a fix(core): avoid filters from overlapping on other pages when changing query params 2025-09-17 10:36:20 +02:00
brian.mulier
887537d8c1 fix(core): avoid clearing filters when reclicking on current left menu item
closes #9476
2025-09-17 10:36:20 +02:00
brian.mulier
0630b741b9 fix(core): avoid undefined error on refresh chart 2025-09-17 10:36:20 +02:00
dependabot[bot]
d2b7e723e1 build(deps): bump io.qameta.allure:allure-bom from 2.29.1 to 2.30.0
Bumps [io.qameta.allure:allure-bom](https://github.com/allure-framework/allure-java) from 2.29.1 to 2.30.0.
- [Release notes](https://github.com/allure-framework/allure-java/releases)
- [Commits](https://github.com/allure-framework/allure-java/compare/2.29.1...2.30.0)

---
updated-dependencies:
- dependency-name: io.qameta.allure:allure-bom
  dependency-version: 2.30.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:22:27 +02:00
dependabot[bot]
080ceadf37 build(deps): bump org.jooq:jooq from 3.20.6 to 3.20.7
Bumps org.jooq:jooq from 3.20.6 to 3.20.7.

---
updated-dependencies:
- dependency-name: org.jooq:jooq
  dependency-version: 3.20.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:21:49 +02:00
dependabot[bot]
a89d902bc2 build(deps): bump andrcuns/allure-publish-action from 2.9.0 to 2.10.0
Bumps [andrcuns/allure-publish-action](https://github.com/andrcuns/allure-publish-action) from 2.9.0 to 2.10.0.
- [Release notes](https://github.com/andrcuns/allure-publish-action/releases)
- [Commits](https://github.com/andrcuns/allure-publish-action/compare/v2.9.0...v2.10.0)

---
updated-dependencies:
- dependency-name: andrcuns/allure-publish-action
  dependency-version: 2.10.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:20:02 +02:00
dependabot[bot]
e2ef7d412a build(deps): bump software.amazon.awssdk.crt:aws-crt
Bumps [software.amazon.awssdk.crt:aws-crt](https://github.com/awslabs/aws-crt-java) from 0.38.11 to 0.38.13.
- [Release notes](https://github.com/awslabs/aws-crt-java/releases)
- [Commits](https://github.com/awslabs/aws-crt-java/compare/v0.38.11...v0.38.13)

---
updated-dependencies:
- dependency-name: software.amazon.awssdk.crt:aws-crt
  dependency-version: 0.38.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:19:13 +02:00
dependabot[bot]
54c667ec4b build(deps): bump com.google.cloud:libraries-bom from 26.67.0 to 26.68.0
Bumps [com.google.cloud:libraries-bom](https://github.com/googleapis/java-cloud-bom) from 26.67.0 to 26.68.0.
- [Release notes](https://github.com/googleapis/java-cloud-bom/releases)
- [Changelog](https://github.com/googleapis/java-cloud-bom/blob/main/release-please-config.json)
- [Commits](https://github.com/googleapis/java-cloud-bom/compare/v26.67.0...v26.68.0)

---
updated-dependencies:
- dependency-name: com.google.cloud:libraries-bom
  dependency-version: 26.68.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-17 10:18:19 +02:00
Piyush Bhaskar
1c53758d33 refactor(core): composable equivalent for mixins (#11277)
* refactor(core): composable equivalent for mixins

* fix: few tweaks

* fix: remove extension
2025-09-17 11:31:20 +05:30
Ludovic DEHON
d092556bc2 chore(build): use remote actions 2025-09-16 18:09:54 +02:00
Roman Acevedo
308106d532 ci: make generated test report retrocompatible with older releases (#11308)
* ci: make generated test report retrocompatible with older realeases

* ci: fix cli
2025-09-16 15:21:56 +02:00
Piyush Bhaskar
8fe8f96278 refactor(core): use el-splitter instead of custom sliders (#11309)
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-16 18:35:57 +05:30
Miloš Paunović
a5cad6d87c chore(core): improve coloring scheme for dependencies graph (#11306) 2025-09-16 14:26:15 +02:00
Loïc Mathieu
199d67fbe2 chore(system): share the application.yaml config file between OSS and EE 2025-09-16 10:53:53 +02:00
Loïc Mathieu
558a2e3f01 fix(flows): properly coompute flow dependencies with preconditions
When both upstream flows and where are set, it should be a AND between the two as dependencies must match the upstream flows.

Fixes #11164
2025-09-16 10:43:55 +02:00
HARSH THAKARE
e1d2c30e54 fix(core): add validation to prevent empty label values in Labels task (#11273)
part of #11227

---------

Co-authored-by: harshinfomaticae <harsh.thakare@infomaticae.co.in>
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-16 10:26:46 +02:00
Loïc Mathieu
700c6de411 fix(system): allow flattening a map with duplicated keys 2025-09-16 10:24:43 +02:00
Florian Hussonnois
2b838a5012 fix(executions): add missing CrudEvent on purge execution
Related-to: kestra-io/kestra-ee#5061
2025-09-16 09:34:19 +02:00
Loïc Mathieu
617daa79db fix(executions): truncate the execution_running table as in 0.24 there was an issue in the purge
This table contains executions for flows that have a concurrency that are currently running.
It has been added in 0.24 but in that release there was a bug that may prevent some records to being correctly removed from this table.
To fix that, we truncate it once.
2025-09-15 17:29:28 +02:00
Roman Acevedo
1791127acb test: unflaky FileChangedEventListener and PluginDefaultServiceTest, debug log on JdbcServiceLivenessCoordinatorTest
* test: parallelize AbstractRunnerTest

* test: add TestsUtils.randomTenant(..) function

* test: i think i found a bug

* revert debug

* test: add comment on potential bug, make test pass

* test: fix test metadata

* test: unflaky PluginDefaultServiceTest by separating class

* test: add log on JdbcServiceLivenessCoordinatorTest to debug

* test: cleanup debug log

* fix
2025-09-15 17:07:37 +02:00
brian-mulier-p
7feb571fb3 fix(test): add tenant-in-path storage test (#11292)
part of kestra-io/storage-s3#166
2025-09-15 16:49:02 +02:00
brian-mulier-p
a315bd0e1c fix(security): enhance basic auth security (#11285)
closes kestra-io/kestra-ee#5111
2025-09-15 16:27:14 +02:00
Roman Acevedo
e2ac1e7e98 ci: prevent commenting PR test report when cancelled 2025-09-15 16:01:07 +02:00
Miloš Paunović
c6f40eff52 fix(core): adjust positioning of default tour elements (#11286)
The problem occurred when `No Code` was selected as the `Default Editor Type` in `Settings`. This `PR` resolves the issue.

Closes https://github.com/kestra-io/kestra/issues/9556.
2025-09-15 14:55:00 +02:00
Miloš Paunović
ccd42f7a1a chore(core): remove superfluous button attribute in settings page (#11283) 2025-09-15 12:27:19 +02:00
Florian Hussonnois
ef08c8ac30 fix(plugins): remove regex validation on version property
Changes:
* Fixes stable method in Version class
* Remove regex validation on 'version' property

Related-to: kestra-io/kestra-ee#5090
2025-09-15 11:54:10 +02:00
github-actions[bot]
7b527c85a9 chore(core): localize to languages other than english (#11280)
Extended localization support by adding translations for multiple languages using English as the base. This enhances accessibility and usability for non-English-speaking users while keeping English as the source reference.

Co-authored-by: GitHub Action <actions@github.com>
2025-09-15 11:09:17 +02:00
Hamza
d121867066 chore(flows): trigger editor autocompletion when backspace is pressed (#10797)
Closes https://github.com/kestra-io/kestra/issues/10776.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-15 11:07:20 +02:00
Roman Acevedo
a084a9f6f0 ci: fix Summary report test path 2025-09-15 10:50:25 +02:00
Karthik D
f6fff11081 chore(core): add reset to defaults option to settings page (#11226)
Closes https://github.com/kestra-io/kestra/issues/10640.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-15 10:45:11 +02:00
Roman Acevedo
3d5015938f ci: add total header to generateTestReportSummary 2025-09-15 10:32:22 +02:00
Florian Hussonnois
951c93cedb fix(core): fix CrudEvent model for DELETE operation
Refactor XxxRepository class to use new factory methods
from the CrudEvent class

Related-to: kestra-io/kestra-ee#4727
2025-09-15 10:06:52 +02:00
Antoine Gauthier
9c06b37989 chore(core): resolve button text overflow on system overview page (#11271)
Closes https://github.com/kestra-io/kestra/issues/11245.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-15 09:57:10 +02:00
Anna Geller
a916a03fdd fix(stats): update edition comparison with latest features and improved descriptions (#11272) 2025-09-14 12:35:26 +02:00
Roman Acevedo
4e728da331 test: disable one last test 2025-09-12 20:24:08 +02:00
Roman Acevedo
166a3932c9 test: do not parallelize yet AbstractRunnerTest 2025-09-12 20:24:08 +02:00
Roman Acevedo
0a21971bbf ci: only comment PR with test report in PR 2025-09-12 20:24:08 +02:00
Roman Acevedo
8c4d7c0f9e test: disable failing tests, they will be fixed soon
- will be treated in https://github.com/kestra-io/kestra/issues/11269
2025-09-12 20:24:08 +02:00
Nicolas K.
b709913071 test: run core tests in parallel (#11265)
- advance on #11264

* feat(ci-cd): play tests in parallel and synchronize plugin registry init

* fix(tests): change memory to h2 because the configuration have changed

* feat(tests): use tenant id to run runner tests in parallel

* run AbstractRunnerTest test methods in parallel

* feat(tests): use tenant id to run runner tests in parallel

* feat(tests): remove unwanted generated files

---------

Co-authored-by: Roman Acevedo <roman.acevedo62@gmail.com>
Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-12 19:29:38 +02:00
Roman Acevedo
5be401d23c ci: add a kestra-devtools cli, and comment PR with failed tests
this is a POC, I think it can already be useful. Next step will be to move kestra-devtools to a separate repo and publish it to npm
2025-09-12 18:48:12 +02:00
Roman Acevedo
bb9f4be8c2 Revert "chore(sanitycheck): refactor PurgeCurrentExecutionFiles (#11115)"
This reverts commit fc690bf7cd.
Python task cannot be used here, it is not available. This commit was
wrongly merged with a red CI
2025-09-12 17:49:02 +02:00
François Delbrayelle
01e8e46b77 Revert "feat(retry): use the retry policy on HttpClient (#10922)" (#11263)
This reverts commit a236688be6.
2025-09-12 17:46:28 +02:00
Miloš Paunović
d00f4b0768 chore(core): ensure editor suggestion widget renders above other elements (#11258)
Closes https://github.com/kestra-io/kestra/issues/10702.
Closes https://github.com/kestra-io/kestra/issues/11033.
2025-09-12 14:48:56 +02:00
Barthélémy Ledoux
279f59c874 fix(core): only display close all tabs when there is more than one tab (#11257) 2025-09-12 14:20:54 +02:00
Barthélémy Ledoux
d897509726 fix(flows): clear tasks list when last task is deleted (#11255) 2025-09-12 14:20:42 +02:00
Pradumna Saraf
0d592342af chore(sanitycheck): add for OutputValues (#11105) 2025-09-12 16:53:13 +05:30
Pradumna Saraf
fc690bf7cd chore(sanitycheck): refactor PurgeCurrentExecutionFiles (#11115) 2025-09-12 16:52:37 +05:30
Antoine Gauthier
0a1b919863 chore(logs): display copy button only on row hover (#11254)
Closes https://github.com/kestra-io/kestra/issues/11220.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-12 12:00:08 +02:00
Piyush Bhaskar
2f4e981a29 fix(core): add gradient at footer to avoid hard cut (#11252) 2025-09-12 14:35:47 +05:30
brian-mulier-p
5e7739432e fix(core): add ability to remap sort keys (#11233)
part of kestra-io/kestra-ee#5075
2025-09-12 09:43:39 +02:00
Miloš Paunović
8aba863b8c feat(core): introduce close all panels functionality (#11225)
Closes https://github.com/kestra-io/kestra/issues/10785.
2025-09-12 09:01:24 +02:00
dependabot[bot]
7eaa43c50f build(deps): bump axios (#11243)
Bumps the npm_and_yarn group with 1 update in the /ui directory: [axios](https://github.com/axios/axios).


Updates `axios` from 1.11.0 to 1.12.0
- [Release notes](https://github.com/axios/axios/releases)
- [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md)
- [Commits](https://github.com/axios/axios/compare/v1.11.0...v1.12.0)

---
updated-dependencies:
- dependency-name: axios
  dependency-version: 1.12.0
  dependency-type: direct:production
  dependency-group: npm_and_yarn
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-09-12 08:36:02 +02:00
Piyush Bhaskar
267ff78bfe fix(admin): change the header and add description on hover (#11241)
Co-authored-by: GitHub Action <actions@github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-12 12:00:41 +05:30
François Delbrayelle
7272cfe01f feat(ai_copilot): gray italic placeholder + rename AiAgent to AiCopilot (#11235) 2025-09-11 20:24:04 +02:00
brian.mulier
91e2fdb2cc fix(ai): increase maxOutputToken default 2025-09-11 18:11:52 +02:00
François Delbrayelle
a236688be6 feat(retry): use the retry policy on HttpClient (#10922) 2025-09-11 15:00:25 +02:00
Antoine Gauthier
81763d40ae fix(docs): center main container in DocsLayout (#11222)
Co-authored-by: Piyush Bhaskar <impiyush0012@gmail.com>
2025-09-11 16:18:12 +05:30
Miloš Paunović
677efb6739 fix(namespaces): open details page at top (#11221)
Closes https://github.com/kestra-io/kestra/issues/10536.
2025-09-11 10:52:47 +02:00
Nicolas K.
b35924fef1 fix(tests): add server type mock in the kestra context (#11176)
Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-11 09:45:51 +02:00
Jaem Dessources
9dd93294b6 fix(core): align copy logs button to each row’s right edge (#11216)
Closes https://github.com/kestra-io/kestra/issues/10898.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-11 08:55:01 +02:00
Piyush Bhaskar
fac6dfe9a0 fix(core): update router usage in loadAutocomplete. (#11219) 2025-09-11 12:13:05 +05:30
Bisesh
3bf9764505 fix(core): make sidebar tab color consistent when unfocused (#11217)
Closes https://github.com/kestra-io/kestra/issues/11156.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-11 08:33:57 +02:00
Piyush Bhaskar
c35cea5d19 fix(core): override the ns module. (#11218) 2025-09-11 11:53:00 +05:30
Barthélémy Ledoux
4d8e9479f1 refactor: finally get rid of vuex (#11211) 2025-09-10 22:44:21 +02:00
Florian Hussonnois
3f24e8e838 fix(core): make CRC32 for plugin JARs lazy
Make CRC32 calculation for lazy plugin JAR files
to avoid excessive startup time and performance impact.

Avoid byte buffer reallocation while computing CRC32.
2025-09-10 17:42:02 +02:00
Miloš Paunović
7175fcb666 fix(executions): refactor link creation to ensure the id is rendered as a clickable link (#11209)
Related to https://github.com/kestra-io/kestra/issues/10906.
2025-09-10 15:01:29 +02:00
Barthélémy Ledoux
2ddfa13b1b refactor: make-axios-composable (#11177) 2025-09-10 14:54:00 +02:00
Barthélémy Ledoux
ba2a5dfec8 chore: revert monaco update (#11207) 2025-09-10 13:34:33 +02:00
Loïc Mathieu
f84441dac7 fix(ci): disable publishing docker image on fork
I should have not trusted an AI for this but copy/paste what I know work: the Quarkus CI!
2025-09-10 12:17:25 +02:00
Barthélémy Ledoux
433b788e4a chore: a bunch of performance fixes detected by oxlint (eslint-unicorn) (#10050) 2025-09-10 11:35:07 +02:00
dependabot[bot]
65c5fd6331 build(deps): bump org.projectlombok:lombok from 1.18.38 to 1.18.40
Bumps [org.projectlombok:lombok](https://github.com/projectlombok/lombok) from 1.18.38 to 1.18.40.
- [Changelog](https://github.com/projectlombok/lombok/blob/master/doc/changelog.markdown)
- [Commits](https://github.com/projectlombok/lombok/compare/v1.18.38...v1.18.40)

---
updated-dependencies:
- dependency-name: org.projectlombok:lombok
  dependency-version: 1.18.40
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 11:12:24 +02:00
dependabot[bot]
421ab40276 build(deps): bump io.micrometer:micrometer-core from 1.15.3 to 1.15.4
Bumps [io.micrometer:micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.15.3 to 1.15.4.
- [Release notes](https://github.com/micrometer-metrics/micrometer/releases)
- [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.15.3...v1.15.4)

---
updated-dependencies:
- dependency-name: io.micrometer:micrometer-core
  dependency-version: 1.15.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 11:11:48 +02:00
dependabot[bot]
efb2779693 build(deps): bump flyingSaucerVersion from 9.13.3 to 10.0.0
Bumps `flyingSaucerVersion` from 9.13.3 to 10.0.0.

Updates `org.xhtmlrenderer:flying-saucer-core` from 9.13.3 to 10.0.0
- [Release notes](https://github.com/flyingsaucerproject/flyingsaucer/releases)
- [Changelog](https://github.com/flyingsaucerproject/flyingsaucer/blob/main/CHANGELOG.md)
- [Commits](https://github.com/flyingsaucerproject/flyingsaucer/compare/v9.13.3...v10.0.0)

Updates `org.xhtmlrenderer:flying-saucer-pdf` from 9.13.3 to 10.0.0
- [Release notes](https://github.com/flyingsaucerproject/flyingsaucer/releases)
- [Changelog](https://github.com/flyingsaucerproject/flyingsaucer/blob/main/CHANGELOG.md)
- [Commits](https://github.com/flyingsaucerproject/flyingsaucer/compare/v9.13.3...v10.0.0)

---
updated-dependencies:
- dependency-name: org.xhtmlrenderer:flying-saucer-core
  dependency-version: 10.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
- dependency-name: org.xhtmlrenderer:flying-saucer-pdf
  dependency-version: 10.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 11:10:59 +02:00
dependabot[bot]
74d371c0ca build(deps): bump com.azure:azure-sdk-bom from 1.2.37 to 1.2.38
Bumps [com.azure:azure-sdk-bom](https://github.com/azure/azure-sdk-for-java) from 1.2.37 to 1.2.38.
- [Release notes](https://github.com/azure/azure-sdk-for-java/releases)
- [Commits](https://github.com/azure/azure-sdk-for-java/compare/azure-sdk-bom_1.2.37...azure-sdk-bom_1.2.38)

---
updated-dependencies:
- dependency-name: com.azure:azure-sdk-bom
  dependency-version: 1.2.38
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 11:10:10 +02:00
Loïc Mathieu
90a7869020 fixsystem): always load netty from the app classloader
As Netty is used in core and a lot of plugins, and we already load project reactor from the app classloader that depends in Netty.

Fixes https://github.com/kestra-io/kestra-ee/issues/5038
2025-09-10 10:50:22 +02:00
dependabot[bot]
d9ccb50b0f build(deps): bump actions/github-script from 7 to 8
Bumps [actions/github-script](https://github.com/actions/github-script) from 7 to 8.
- [Release notes](https://github.com/actions/github-script/releases)
- [Commits](https://github.com/actions/github-script/compare/v7...v8)

---
updated-dependencies:
- dependency-name: actions/github-script
  dependency-version: '8'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:47:40 +02:00
dependabot[bot]
aea0b87ef8 build(deps): bump aquasecurity/trivy-action from 0.33.0 to 0.33.1
Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.33.0 to 0.33.1.
- [Release notes](https://github.com/aquasecurity/trivy-action/releases)
- [Commits](https://github.com/aquasecurity/trivy-action/compare/0.33.0...0.33.1)

---
updated-dependencies:
- dependency-name: aquasecurity/trivy-action
  dependency-version: 0.33.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:47:17 +02:00
Loïc Mathieu
9a144fc3fe fix(system): we don't need to advance the parser anymore to the first token 2025-09-10 10:46:44 +02:00
Loïc Mathieu
ddd9cebc63 chore(deps): upgrade to Jackson 2.20.0
Jackson annotation now uses a version scheme without micro version so it has been updated to 2.20.

Closes #11069
2025-09-10 10:46:44 +02:00
dependabot[bot]
1bebbb9b73 build(deps): bump com.gorylenko.gradle-git-properties
Bumps com.gorylenko.gradle-git-properties from 2.5.2 to 2.5.3.

---
updated-dependencies:
- dependency-name: com.gorylenko.gradle-git-properties
  dependency-version: 2.5.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:46:26 +02:00
dependabot[bot]
8de4dc867e build(deps): bump actions/setup-python from 5 to 6
Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:46:08 +02:00
dependabot[bot]
fc49694e76 build(deps): bump actions/setup-node from 4 to 5
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 5.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/setup-node
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:45:47 +02:00
dependabot[bot]
152300abae build(deps): bump io.micronaut.openapi:micronaut-openapi-bom
Bumps [io.micronaut.openapi:micronaut-openapi-bom](https://github.com/micronaut-projects/micronaut-openapi) from 6.17.3 to 6.18.0.
- [Release notes](https://github.com/micronaut-projects/micronaut-openapi/releases)
- [Commits](https://github.com/micronaut-projects/micronaut-openapi/compare/v6.17.3...v6.18.0)

---
updated-dependencies:
- dependency-name: io.micronaut.openapi:micronaut-openapi-bom
  dependency-version: 6.18.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:45:13 +02:00
dependabot[bot]
1ff5dda4e1 build(deps): bump software.amazon.awssdk:bom from 2.33.2 to 2.33.5
Bumps software.amazon.awssdk:bom from 2.33.2 to 2.33.5.

---
updated-dependencies:
- dependency-name: software.amazon.awssdk:bom
  dependency-version: 2.33.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-10 10:44:50 +02:00
Miloš Paunović
84f9b8876d chore(deps): regular dependency update (#11200)
Performing a weekly round of dependency updates in the NPM ecosystem to keep everything up to date.
2025-09-10 10:18:33 +02:00
brian-mulier-p
575955567f fix(flows): avoid failing flow dependencies with dynamic defaults (#11166)
closes #11117
2025-09-10 09:59:51 +02:00
brian-mulier-p
d6d2580b45 fix(namespaces): avoid adding 'company.team' as default ns (#11174)
closes #11168
2025-09-09 17:13:48 +02:00
Miloš Paunović
070e54b902 chore(flows): display correct flow dependency count (#11169)
Closes https://github.com/kestra-io/kestra/issues/11127.
2025-09-09 13:56:17 +02:00
Roman Acevedo
829ca4380f fix(flows): topology would not load when having many flows and cyclic relations
- this will probably fix https://github.com/kestra-io/kestra-ee/issues/4980

the issue was recursiveFlowTopology was returning a lot of duplicates, it was aggravated when having many Flows and multiple Flow triggers
2025-09-09 13:06:20 +02:00
Karthik D
381c7a75ad chore(core): use simple search input on blueprints listing (#11034)
Closes https://github.com/kestra-io/kestra/issues/11002.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-09 12:54:58 +02:00
louispy
1688c489a9 chore(flows): improve visibility of horizontal scroll bar on listing (#11163)
Closes https://github.com/kestra-io/kestra/issues/11158.

Co-authored-by: louispy <louisleslie98@gmail.com>
Co-authored-by: MilosPaunovic <paun992@hotmail.com>
2025-09-09 12:40:28 +02:00
AKSHAT GUPTA
93ccbf5f9b chore(core): separate data loading from graph node rendering on dependency view (#11155)
Relates to https://github.com/kestra-io/kestra/issues/11125.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-09 12:25:58 +02:00
Barthélémy Ledoux
ac1cb235e5 refactor: avoid importing all of lodash when we only need groupBy (#10870) 2025-09-09 11:34:13 +02:00
dependabot[bot]
9d3d3642e8 build(deps): bump kafkaVersion from 4.0.0 to 4.1.0
Bumps `kafkaVersion` from 4.0.0 to 4.1.0.

Updates `org.apache.kafka:kafka-clients` from 4.0.0 to 4.1.0

Updates `org.apache.kafka:kafka-streams` from 4.0.0 to 4.1.0

Updates `org.apache.kafka:kafka-streams-test-utils` from 4.0.0 to 4.1.0

---
updated-dependencies:
- dependency-name: org.apache.kafka:kafka-clients
  dependency-version: 4.1.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
- dependency-name: org.apache.kafka:kafka-streams
  dependency-version: 4.1.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
- dependency-name: org.apache.kafka:kafka-streams-test-utils
  dependency-version: 4.1.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-09-09 09:56:38 +02:00
Suguresh
3d306a885e feat(core): add extra date format options (#10237)
Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-09 09:31:49 +02:00
Antoine Gauthier
ef193c5774 feat(core): add a new date format option with milliseconds (#11108)
Closes https://github.com/kestra-io/kestra/issues/11028.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-09 09:20:37 +02:00
AmbarMishra973
d0f46169f4 feat(executions): make the id field a link that can be opened in a new tab (#10963)
Closes https://github.com/kestra-io/kestra/issues/10906.

Co-authored-by: Miloš Paunović <paun992@hotmail.com>
2025-09-09 09:13:49 +02:00
François Delbrayelle
3005ab527c fix(outputs): open external file was not working (#11154) 2025-09-08 17:45:19 +02:00
Barthélémy Ledoux
688e2af12b chore: update eslint config for vue files (#9891) 2025-09-08 16:42:33 +02:00
Nicolas K.
4c0a05f484 fix(test): flaky Scheduler trigger change test (#11153)
Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
2025-09-08 16:33:23 +02:00
zaib shamsi
108f8fc2c7 feat(executions): nicer exception message for the HttpFunction
### What I did

- Improved the exception message in HttpFunction.java to make debugging easier.

### Why

- The original message was too generic. This change makes it clearer where the issue occurs.
2025-09-08 15:04:12 +02:00
Barthélémy Ledoux
8b81a37559 refactor: make folder structure of no-code use "no-code" (#11122) 2025-09-08 14:15:04 +02:00
Barthélémy Ledoux
9222f97d63 fix(core): multipanel split creates super big panels (#11123) 2025-09-08 14:14:40 +02:00
brian.mulier
43e3591417 chore(ci): fail-safe update-plugin-kestra-version.sh 2025-09-08 12:02:28 +02:00
brian.mulier
438dc9ecf6 chore(ci): create branch if not exist on update-plugin-kestra-version.sh 2025-09-08 11:45:15 +02:00
brian-mulier-p
7292837c58 chore(ci): add LTS tagging (#11131) 2025-09-08 11:13:16 +02:00
brian.mulier
7fa93d7764 chore(version): update to version 'v1.1.0-SNAPSHOT'. 2025-09-08 10:08:34 +02:00
622 changed files with 11131 additions and 10901 deletions

View File

@@ -26,7 +26,7 @@ jobs:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@v6
with:
python-version: "3.x"
@@ -39,7 +39,7 @@ jobs:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- name: Set up Node
uses: actions/setup-node@v4
uses: actions/setup-node@v5
with:
node-version: "20.x"

View File

@@ -37,7 +37,7 @@ jobs:
path: kestra
# Setup build
- uses: kestra-io/actions/.github/actions/setup-build@main
- uses: kestra-io/actions/composite/setup-build@main
name: Setup - Build
id: build
with:

View File

@@ -25,21 +25,13 @@ jobs:
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v5
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
- uses: kestra-io/actions/composite/setup-build@main
id: build
with:
java-enabled: true
node-enabled: true
python-enabled: true
caches-enabled: true
# Get Plugins List
- name: Get Plugins List
@@ -60,7 +52,7 @@ jobs:
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
run: |
chmod +x ./dev-tools/release-plugins.sh;
./dev-tools/release-plugins.sh \
--release-version=${{github.event.inputs.releaseVersion}} \
--next-version=${{github.event.inputs.nextVersion}} \
@@ -73,7 +65,7 @@ jobs:
GITHUB_PAT: ${{ secrets.GH_PERSONAL_TOKEN }}
run: |
chmod +x ./dev-tools/release-plugins.sh;
./dev-tools/release-plugins.sh \
--release-version=${{github.event.inputs.releaseVersion}} \
--next-version=${{github.event.inputs.nextVersion}} \

View File

@@ -38,15 +38,8 @@ jobs:
fetch-depth: 0
path: kestra
# Checkout GitHub Actions
- uses: actions/checkout@v5
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
- uses: kestra-io/actions/composite/setup-build@main
id: build
with:
java-enabled: true

View File

@@ -59,8 +59,6 @@ jobs:
needs:
- release
if: always()
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
steps:
- name: Trigger EE Workflow
uses: peter-evans/repository-dispatch@v3
@@ -70,14 +68,9 @@ jobs:
repository: kestra-io/kestra-ee
event-type: "oss-updated"
# Slack
- name: Slack - Notification
uses: Gamesight/slack-workflow-status@master
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') }}
uses: kestra-io/actions/composite/slack-status@main
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
name: GitHub Actions
icon_emoji: ":github-actions:"
channel: "C02DQ1A7JLR" # _int_git channel
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -4,6 +4,7 @@ on:
pull_request:
branches:
- develop
- releases/*
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}-pr
@@ -60,19 +61,3 @@ jobs:
name: E2E - Tests
uses: ./.github/workflows/e2e.yml
end:
name: End
runs-on: ubuntu-latest
if: always()
needs: [frontend, backend]
steps:
# Slack
- name: Slack notification
uses: Gamesight/slack-workflow-status@master
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
name: GitHub Actions
icon_emoji: ":github-actions:"
channel: "C02DQ1A7JLR"

View File

@@ -21,13 +21,6 @@ jobs:
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v5
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
id: build
@@ -70,15 +63,8 @@ jobs:
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v5
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
- uses: kestra-io/actions/composite/setup-build@main
id: build
with:
java-enabled: false
@@ -87,7 +73,7 @@ jobs:
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
- name: Docker Vulnerabilities Check
uses: aquasecurity/trivy-action@0.33.0
uses: aquasecurity/trivy-action@0.33.1
with:
image-ref: kestra/kestra:develop
format: 'template'
@@ -115,24 +101,16 @@ jobs:
with:
fetch-depth: 0
# Checkout GitHub Actions
- uses: actions/checkout@v5
with:
repository: kestra-io/actions
path: actions
ref: main
# Setup build
- uses: ./actions/.github/actions/setup-build
- uses: kestra-io/actions/composite/setup-build@main
id: build
with:
java-enabled: false
node-enabled: false
caches-enabled: true
# Run Trivy image scan for Docker vulnerabilities, see https://github.com/aquasecurity/trivy-action
- name: Docker Vulnerabilities Check
uses: aquasecurity/trivy-action@0.33.0
uses: aquasecurity/trivy-action@0.33.1
with:
image-ref: kestra/kestra:latest
format: table

View File

@@ -20,6 +20,7 @@ permissions:
contents: write
checks: write
actions: read
pull-requests: write
jobs:
test:
@@ -35,7 +36,7 @@ jobs:
fetch-depth: 0
# Setup build
- uses: kestra-io/actions/.github/actions/setup-build@main
- uses: kestra-io/actions/composite/setup-build@main
name: Setup - Build
id: build
with:
@@ -59,84 +60,15 @@ jobs:
export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
./gradlew check javadoc --parallel
# report test
- name: Test - Publish Test Results
uses: dorny/test-reporter@v2
if: always()
with:
name: Java Tests Report
reporter: java-junit
path: '**/build/test-results/test/TEST-*.xml'
list-suites: 'failed'
list-tests: 'failed'
fail-on-error: 'false'
token: ${{ secrets.GITHUB_AUTH_TOKEN }}
# Sonar
- name: Test - Analyze with Sonar
if: env.SONAR_TOKEN != ''
- name: comment PR with test report
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
shell: bash
run: ./gradlew sonar --info
run: npx --yes @kestra-io/kestra-devtools generateTestReportSummary --only-errors --ci $(pwd)
# GCP
- name: GCP - Auth with unit test account
id: auth
if: always() && env.GOOGLE_SERVICE_ACCOUNT != ''
continue-on-error: true
uses: "google-github-actions/auth@v3"
with:
credentials_json: "${{ secrets.GOOGLE_SERVICE_ACCOUNT }}"
- name: GCP - Setup Cloud SDK
if: env.GOOGLE_SERVICE_ACCOUNT != ''
uses: "google-github-actions/setup-gcloud@v3"
# Allure check
- uses: rlespinasse/github-slug-action@v5
name: Allure - Generate slug variables
- name: Allure - Publish report
uses: andrcuns/allure-publish-action@v2.9.0
if: always() && env.GOOGLE_SERVICE_ACCOUNT != ''
continue-on-error: true
env:
GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_AUTH_TOKEN }}
JAVA_HOME: /usr/lib/jvm/default-jvm/
with:
storageType: gcs
resultsGlob: "**/build/allure-results"
bucket: internal-kestra-host
baseUrl: "https://internal.dev.kestra.io"
prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }}
copyLatest: true
ignoreMissingResults: true
# Jacoco
- name: Jacoco - Copy reports
if: env.GOOGLE_SERVICE_ACCOUNT != ''
continue-on-error: true
shell: bash
run: |
mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/
mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml
gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }}
# Codecov
- name: Codecov - Upload coverage reports
uses: codecov/codecov-action@v5
# Report Java
- name: Report - Java
uses: kestra-io/actions/composite/report-java@main
if: ${{ !cancelled() }}
continue-on-error: true
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: backend
- name: Codecov - Upload test results
uses: codecov/test-results-action@v1
if: ${{ !cancelled() }}
continue-on-error: true
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: backend
secrets: ${{ toJSON(secrets) }}

View File

@@ -26,7 +26,7 @@ jobs:
run: npm ci
# Setup build
- uses: kestra-io/actions/.github/actions/setup-build@main
- uses: kestra-io/actions/composite/setup-build@main
name: Setup - Build
id: build
with:

View File

@@ -25,15 +25,6 @@ jobs:
fetch-depth: 0
submodules: true
# Checkout GitHub Actions
- name: Checkout - Actions
uses: actions/checkout@v5
with:
repository: kestra-io/actions
sparse-checkout-cone-mode: true
path: actions
sparse-checkout: |
.github/actions
# Download Exec
# Must be done after checkout actions
@@ -59,7 +50,7 @@ jobs:
# GitHub Release
- name: Create GitHub release
uses: ./actions/.github/actions/github-release
uses: kestra-io/actions/composite/github-release@main
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
MAKE_LATEST: ${{ steps.is_latest.outputs.latest }}
@@ -82,7 +73,7 @@ jobs:
- name: Merge Release Notes
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
uses: ./actions/.github/actions/github-release-note-merge
uses: kestra-io/actions/composite/github-release-note-merge@main
env:
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
RELEASE_TAG: ${{ github.ref_name }}

View File

@@ -11,6 +11,14 @@ on:
options:
- "true"
- "false"
retag-lts:
description: 'Retag LTS Docker images'
required: true
type: choice
default: "false"
options:
- "true"
- "false"
release-tag:
description: 'Kestra Release Tag (by default, deduced with the ref)'
required: false
@@ -179,6 +187,11 @@ jobs:
run: |
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest{0}', matrix.image.name) }}
- name: Retag to LTS
if: startsWith(github.ref, 'refs/tags/v') && inputs.retag-lts == 'true'
run: |
regctl image copy ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }} ${{ format('kestra/kestra:latest-lts{0}', matrix.image.name) }}
end:
runs-on: ubuntu-latest
needs:
@@ -187,14 +200,9 @@ jobs:
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
steps:
# Slack
- name: Slack notification
uses: Gamesight/slack-workflow-status@master
if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
if: ${{ failure() && env.SLACK_WEBHOOK_URL != 0 }}
uses: kestra-io/actions/composite/slack-status@main
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
name: GitHub Actions
icon_emoji: ':github-actions:'
channel: 'C02DQ1A7JLR' # _int_git channel
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -29,7 +29,7 @@ jobs:
# Setup build
- name: Setup - Build
uses: kestra-io/actions/.github/actions/setup-build@main
uses: kestra-io/actions/composite/setup-build@main
id: build
with:
java-enabled: true

View File

@@ -7,7 +7,7 @@ on:
jobs:
publish:
name: Pull Request - Delete Docker
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
if: github.repository == 'kestra-io/kestra' # prevent running on forks
runs-on: ubuntu-latest
steps:
- uses: dataaxiom/ghcr-cleanup-action@v1

View File

@@ -8,12 +8,12 @@ on:
jobs:
build-artifacts:
name: Build Artifacts
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
if: github.repository == 'kestra-io/kestra' # prevent running on forks
uses: ./.github/workflows/workflow-build-artifacts.yml
publish:
name: Publish Docker
if: github.repository == github.event.pull_request.head.repo.full_name # prevent running on forks
if: github.repository == 'kestra-io/kestra' # prevent running on forks
runs-on: ubuntu-latest
needs: build-artifacts
env:
@@ -62,7 +62,7 @@ jobs:
# Add comment on pull request
- name: Add comment to PR
uses: actions/github-script@v7
uses: actions/github-script@v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |

View File

@@ -84,14 +84,12 @@ jobs:
name: Notify - Slack
runs-on: ubuntu-latest
needs: [ frontend, backend ]
if: github.event_name == 'schedule'
steps:
- name: Notify failed CI
id: send-ci-failed
if: |
always() && (needs.frontend.result != 'success' ||
needs.backend.result != 'success')
uses: kestra-io/actions/.github/actions/send-ci-failed@main
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
always() &&
(needs.frontend.result != 'success' || needs.backend.result != 'success') &&
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')
uses: kestra-io/actions/composite/slack-status@main
with:
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -33,10 +33,10 @@
<p align="center">
<a href="https://go.kestra.io/video/product-overview" target="_blank">
<img src="https://kestra.io/startvideo.png" alt="Get started in 4 minutes with Kestra" width="640px" />
<img src="https://kestra.io/startvideo.png" alt="Get started in 3 minutes with Kestra" width="640px" />
</a>
</p>
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 4 minutes.</i></p>
<p align="center" style="color:grey;"><i>Click on the image to learn how to get started with Kestra in 3 minutes.</i></p>
## 🌟 What is Kestra?

View File

@@ -32,12 +32,12 @@ plugins {
// release
id 'net.researchgate.release' version '3.1.0'
id "com.gorylenko.gradle-git-properties" version "2.5.2"
id "com.gorylenko.gradle-git-properties" version "2.5.3"
id 'signing'
id "com.vanniktech.maven.publish" version "0.34.0"
// OWASP dependency check
id "org.owasp.dependencycheck" version "12.1.3" apply false
id "org.owasp.dependencycheck" version "12.1.5" apply false
}
idea {
@@ -168,8 +168,9 @@ allprojects {
/**********************************************************************************************************************\
* Test
**********************************************************************************************************************/
subprojects {
if (it.name != 'platform' && it.name != 'jmh-benchmarks') {
subprojects {subProj ->
if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') {
apply plugin: "com.adarshr.test-logger"
java {
@@ -207,6 +208,13 @@ subprojects {
test {
useJUnitPlatform()
reports {
junitXml.required = true
junitXml.outputPerTestCase = true
junitXml.mergeReruns = true
junitXml.includeSystemErrLog = true;
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
}
// set Xmx for test workers
maxHeapSize = '4g'
@@ -222,6 +230,15 @@ subprojects {
environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
environment 'ENV_TEST1', "true"
environment 'ENV_TEST2', "Pass by env"
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
// JUnit 5 parallel settings
systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
}
}
testlogger {

View File

@@ -40,5 +40,6 @@ dependencies {
implementation project(":worker")
//test
testImplementation project(':tests')
testImplementation "org.wiremock:wiremock-jetty12"
}

View File

@@ -49,7 +49,7 @@ import java.util.concurrent.Callable;
@Introspected
public class App implements Callable<Integer> {
public static void main(String[] args) {
execute(App.class, args);
execute(App.class, new String [] { Environment.CLI }, args);
}
@Override
@@ -57,13 +57,13 @@ public class App implements Callable<Integer> {
return PicocliRunner.call(App.class, "--help");
}
protected static void execute(Class<?> cls, String... args) {
protected static void execute(Class<?> cls, String[] environments, String... args) {
// Log Bridge
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
// Init ApplicationContext
ApplicationContext applicationContext = App.applicationContext(cls, args);
ApplicationContext applicationContext = App.applicationContext(cls, environments, args);
// Call Picocli command
int exitCode = 0;
@@ -80,6 +80,7 @@ public class App implements Callable<Integer> {
System.exit(Objects.requireNonNullElse(exitCode, 0));
}
/**
* Create an {@link ApplicationContext} with additional properties based on configuration files (--config) and
* forced Properties from current command.
@@ -88,12 +89,13 @@ public class App implements Callable<Integer> {
* @return the application context created
*/
protected static ApplicationContext applicationContext(Class<?> mainClass,
String[] environments,
String[] args) {
ApplicationContextBuilder builder = ApplicationContext
.builder()
.mainClass(mainClass)
.environments(Environment.CLI);
.environments(environments);
CommandLine cmd = new CommandLine(mainClass, CommandLine.defaultFactory());
continueOnParsingErrors(cmd);

View File

@@ -2,19 +2,27 @@ package io.kestra.cli.commands.servers;
import io.kestra.cli.AbstractCommand;
import io.kestra.core.contexts.KestraContext;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;
abstract public class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
@Slf4j
public abstract class AbstractServerCommand extends AbstractCommand implements ServerCommandInterface {
@CommandLine.Option(names = {"--port"}, description = "The port to bind")
Integer serverPort;
@Override
public Integer call() throws Exception {
log.info("Machine information: {} available cpu(s), {}MB max memory, Java version {}", Runtime.getRuntime().availableProcessors(), maxMemoryInMB(), Runtime.version());
this.shutdownHook(true, () -> KestraContext.getContext().shutdown());
return super.call();
}
private long maxMemoryInMB() {
return Runtime.getRuntime().maxMemory() / 1024 / 1024;
}
protected static int defaultWorkerThread() {
return Runtime.getRuntime().availableProcessors() * 8;
}

View File

@@ -262,6 +262,8 @@ public class FileChangedEventListener {
}
private String getTenantIdFromPath(Path path) {
// FIXME there is probably a bug here when a tenant has '_' in its name,
// a valid tenant name is defined with following regex: "^[a-z0-9][a-z0-9_-]*"
return path.getFileName().toString().split("_")[0];
}
}

View File

@@ -167,6 +167,8 @@ kestra:
open-urls:
- "/ping"
- "/api/v1/executions/webhook/"
- "/api/v1/main/executions/webhook/"
- "/api/v1/*/executions/webhook/"
preview:
initial-rows: 100

View File

@@ -37,7 +37,7 @@ class AppTest {
final String[] args = new String[]{"server", serverType, "--help"};
try (ApplicationContext ctx = App.applicationContext(App.class, args)) {
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, args)) {
new CommandLine(App.class, new MicronautFactory(ctx)).execute(args);
assertTrue(ctx.getProperty("kestra.server-type", ServerType.class).isEmpty());
@@ -52,7 +52,7 @@ class AppTest {
final String[] argsWithMissingParams = new String[]{"flow", "namespace", "update"};
try (ApplicationContext ctx = App.applicationContext(App.class, argsWithMissingParams)) {
try (ApplicationContext ctx = App.applicationContext(App.class, new String [] { Environment.CLI }, argsWithMissingParams)) {
new CommandLine(App.class, new MicronautFactory(ctx)).execute(argsWithMissingParams);
assertThat(out.toString()).startsWith("Missing required parameters: ");

View File

@@ -4,11 +4,11 @@ import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.GenericFlow;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.utils.Await;
import io.kestra.core.utils.TestsUtils;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.*;
import org.junitpioneer.jupiter.RetryingTest;
import java.io.IOException;
import java.nio.file.Files;
@@ -18,8 +18,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junitpioneer.jupiter.RetryingTest;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static io.kestra.core.utils.Rethrow.throwRunnable;
import static org.assertj.core.api.Assertions.assertThat;
@@ -57,10 +57,11 @@ class FileChangedEventListenerTest {
}
}
@RetryingTest(5) // Flaky on CI but always pass locally
@Test
void test() throws IOException, TimeoutException {
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getSimpleName(), "test");
// remove the flow if it already exists
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "myflow").ifPresent(flow -> flowRepository.delete(flow));
// create a basic flow
String flow = """
@@ -73,14 +74,14 @@ class FileChangedEventListenerTest {
message: Hello World! 🚀
""";
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, flow);
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, flow);
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), flow.getBytes());
Await.until(
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isPresent(),
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isPresent(),
Duration.ofMillis(100),
Duration.ofSeconds(10)
);
Flow myflow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").orElseThrow();
Flow myflow = flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").orElseThrow();
assertThat(myflow.getTasks()).hasSize(1);
assertThat(myflow.getTasks().getFirst().getId()).isEqualTo("hello");
assertThat(myflow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
@@ -88,16 +89,17 @@ class FileChangedEventListenerTest {
// delete the flow
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
Await.until(
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "myflow").isEmpty(),
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "myflow").isEmpty(),
Duration.ofMillis(100),
Duration.ofSeconds(10)
);
}
@RetryingTest(5) // Flaky on CI but always pass locally
@RetryingTest(2)
void testWithPluginDefault() throws IOException, TimeoutException {
var tenant = TestsUtils.randomTenant(FileChangedEventListenerTest.class.getName(), "testWithPluginDefault");
// remove the flow if it already exists
flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
flowRepository.findByIdWithSource(tenant, "io.kestra.tests.watch", "pluginDefault").ifPresent(flow -> flowRepository.delete(flow));
// create a flow with plugin default
String pluginDefault = """
@@ -113,14 +115,14 @@ class FileChangedEventListenerTest {
values:
message: Hello World!
""";
GenericFlow genericFlow = GenericFlow.fromYaml(MAIN_TENANT, pluginDefault);
GenericFlow genericFlow = GenericFlow.fromYaml(tenant, pluginDefault);
Files.write(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"), pluginDefault.getBytes());
Await.until(
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isPresent(),
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isPresent(),
Duration.ofMillis(100),
Duration.ofSeconds(10)
);
Flow pluginDefaultFlow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
Flow pluginDefaultFlow = flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").orElseThrow();
assertThat(pluginDefaultFlow.getTasks()).hasSize(1);
assertThat(pluginDefaultFlow.getTasks().getFirst().getId()).isEqualTo("helloWithDefault");
assertThat(pluginDefaultFlow.getTasks().getFirst().getType()).isEqualTo("io.kestra.plugin.core.log.Log");
@@ -128,7 +130,7 @@ class FileChangedEventListenerTest {
// delete both files
Files.delete(Path.of(FILE_WATCH + "/" + genericFlow.uidWithoutRevision() + ".yaml"));
Await.until(
() -> flowRepository.findById(MAIN_TENANT, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
() -> flowRepository.findById(tenant, "io.kestra.tests.watch", "pluginDefault").isEmpty(),
Duration.ofMillis(100),
Duration.ofSeconds(10)
);

View File

@@ -84,7 +84,7 @@ dependencies {
testImplementation "org.testcontainers:testcontainers:1.21.3"
testImplementation "org.testcontainers:junit-jupiter:1.21.3"
testImplementation "org.bouncycastle:bcpkix-jdk18on:1.81"
testImplementation "org.bouncycastle:bcpkix-jdk18on"
testImplementation "org.wiremock:wiremock-jetty12"
}

View File

@@ -3,30 +3,88 @@ package io.kestra.core.events;
import io.micronaut.core.annotation.Nullable;
import io.micronaut.http.HttpRequest;
import io.micronaut.http.context.ServerRequestContext;
import lombok.AllArgsConstructor;
import lombok.Getter;
@AllArgsConstructor
import java.util.Objects;
@Getter
public class CrudEvent<T> {
T model;
private final T model;
@Nullable
T previousModel;
CrudEventType type;
HttpRequest<?> request;
private final T previousModel;
private final CrudEventType type;
private final HttpRequest<?> request;
/**
* Static helper method for creating a new {@link CrudEventType#UPDATE} CrudEvent.
*
* @param model the new created model.
* @param <T> type of the model.
* @return the new {@link CrudEvent}.
*/
public static <T> CrudEvent<T> create(T model) {
Objects.requireNonNull(model, "Can't create CREATE event with a null model");
return new CrudEvent<>(model, null, CrudEventType.CREATE);
}
/**
* Static helper method for creating a new {@link CrudEventType#DELETE} CrudEvent.
*
* @param model the deleted model.
* @param <T> type of the model.
* @return the new {@link CrudEvent}.
*/
public static <T> CrudEvent<T> delete(T model) {
Objects.requireNonNull(model, "Can't create DELETE event with a null model");
return new CrudEvent<>(null, model, CrudEventType.DELETE);
}
/**
* Static helper method for creating a new CrudEvent.
*
* @param before the model before the update.
* @param after the model after the update.
* @param <T> type of the model.
* @return the new {@link CrudEvent}.
*/
public static <T> CrudEvent<T> of(T before, T after) {
if (before == null && after == null) {
throw new IllegalArgumentException("Both before and after cannot be null");
}
if (before == null) {
return create(after);
}
if (after == null) {
return delete(before);
}
return new CrudEvent<>(after, before, CrudEventType.UPDATE);
}
/**
* @deprecated use the static factory methods.
*/
@Deprecated
public CrudEvent(T model, CrudEventType type) {
this.model = model;
this.type = type;
this.previousModel = null;
this.request = ServerRequestContext.currentRequest().orElse(null);
this(
CrudEventType.DELETE.equals(type) ? null : model,
CrudEventType.DELETE.equals(type) ? model : null,
type,
ServerRequestContext.currentRequest().orElse(null)
);
}
public CrudEvent(T model, T previousModel, CrudEventType type) {
this(model, previousModel, type, ServerRequestContext.currentRequest().orElse(null));
}
public CrudEvent(T model, T previousModel, CrudEventType type, HttpRequest<?> request) {
this.model = model;
this.previousModel = previousModel;
this.type = type;
this.request = ServerRequestContext.currentRequest().orElse(null);
this.request = request;
}
}

View File

@@ -2,12 +2,13 @@ package io.kestra.core.models;
import io.kestra.core.utils.MapUtils;
import jakarta.annotation.Nullable;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.NotEmpty;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public record Label(@NotNull String key, @NotNull String value) {
public record Label(@NotEmpty String key, @NotEmpty String value) {
public static final String SYSTEM_PREFIX = "system.";
// system labels
@@ -41,7 +42,7 @@ public record Label(@NotNull String key, @NotNull String value) {
public static Map<String, String> toMap(@Nullable List<Label> labels) {
if (labels == null || labels.isEmpty()) return Collections.emptyMap();
return labels.stream()
.filter(label -> label.value() != null && label.key() != null)
.filter(label -> label.value() != null && !label.value().isEmpty() && label.key() != null && !label.key().isEmpty())
// using an accumulator in case labels with the same key exists: the second is kept
.collect(Collectors.toMap(Label::key, Label::value, (first, second) -> second, LinkedHashMap::new));
}
@@ -56,6 +57,7 @@ public record Label(@NotNull String key, @NotNull String value) {
public static List<Label> deduplicate(@Nullable List<Label> labels) {
if (labels == null || labels.isEmpty()) return Collections.emptyList();
return toMap(labels).entrySet().stream()
.filter(getEntryNotEmptyPredicate())
.map(entry -> new Label(entry.getKey(), entry.getValue()))
.collect(Collectors.toCollection(ArrayList::new));
}
@@ -70,6 +72,7 @@ public record Label(@NotNull String key, @NotNull String value) {
if (map == null || map.isEmpty()) return List.of();
return map.entrySet()
.stream()
.filter(getEntryNotEmptyPredicate())
.map(entry -> new Label(entry.getKey(), entry.getValue()))
.toList();
}
@@ -88,4 +91,14 @@ public record Label(@NotNull String key, @NotNull String value) {
}
return map;
}
/**
* Provides predicate for not empty entries.
*
* @return The non-empty filter
*/
public static Predicate<Map.Entry<String, String>> getEntryNotEmptyPredicate() {
return entry -> entry.getKey() != null && !entry.getKey().isEmpty() &&
entry.getValue() != null && !entry.getValue().isEmpty();
}
}

View File

@@ -1,16 +1,33 @@
package io.kestra.core.models;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.Pattern;
import java.util.List;
import java.util.Map;
/**
* Interface that can be implemented by classes supporting plugin versioning.
*
* @see Plugin
*/
public interface PluginVersioning {
String TITLE = "Plugin Version";
String DESCRIPTION = """
Defines the version of the plugin to use.
@Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)")
@Schema(title = "The version of the plugin to use.")
The version must follow the Semantic Versioning (SemVer) specification:
- A single-digit MAJOR version (e.g., `1`).
- A MAJOR.MINOR version (e.g., `1.1`).
- A MAJOR.MINOR.PATCH version, optionally with any qualifier
(e.g., `1.1.2`, `1.1.0-SNAPSHOT`).
""";
@Schema(
title = TITLE,
description = DESCRIPTION
)
String getVersion();
}

View File

@@ -254,19 +254,7 @@ public record QueryFilter(
*
* @return List of {@code ResourceField} with resource names, fields, and operations.
*/
public static List<ResourceField> asResourceList() {
return Arrays.stream(values())
.map(Resource::toResourceField)
.toList();
}
private static ResourceField toResourceField(Resource resource) {
List<FieldOp> fieldOps = resource.supportedField().stream()
.map(Resource::toFieldInfo)
.toList();
return new ResourceField(resource.name().toLowerCase(), fieldOps);
}
private static FieldOp toFieldInfo(Field field) {
List<Operation> operations = field.supportedOp().stream()
.map(Resource::toOperation)
@@ -279,9 +267,6 @@ public record QueryFilter(
}
}
public record ResourceField(String name, List<FieldOp> fields) {
}
public record FieldOp(String name, String value, List<Operation> operations) {
}

View File

@@ -17,31 +17,12 @@ import java.util.List;
@Introspected
public class ExecutionUsage {
private final List<DailyExecutionStatistics> dailyExecutionsCount;
private final List<DailyExecutionStatistics> dailyTaskRunsCount;
public static ExecutionUsage of(final String tenantId,
final ExecutionRepositoryInterface executionRepository,
final ZonedDateTime from,
final ZonedDateTime to) {
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
try {
dailyTaskRunsCount = executionRepository.dailyStatistics(
null,
tenantId,
null,
null,
null,
from,
to,
DateUtils.GroupType.DAY,
null,
true);
} catch (UnsupportedOperationException ignored) {
}
return ExecutionUsage.builder()
.dailyExecutionsCount(executionRepository.dailyStatistics(
null,
@@ -52,28 +33,13 @@ public class ExecutionUsage {
from,
to,
DateUtils.GroupType.DAY,
null,
false))
.dailyTaskRunsCount(dailyTaskRunsCount)
null))
.build();
}
public static ExecutionUsage of(final ExecutionRepositoryInterface repository,
final ZonedDateTime from,
final ZonedDateTime to) {
List<DailyExecutionStatistics> dailyTaskRunsCount = null;
try {
dailyTaskRunsCount = repository.dailyStatisticsForAllTenants(
null,
null,
null,
from,
to,
DateUtils.GroupType.DAY,
true
);
} catch (UnsupportedOperationException ignored) {}
return ExecutionUsage.builder()
.dailyExecutionsCount(repository.dailyStatisticsForAllTenants(
null,
@@ -81,10 +47,8 @@ public class ExecutionUsage {
null,
from,
to,
DateUtils.GroupType.DAY,
false
DateUtils.GroupType.DAY
))
.dailyTaskRunsCount(dailyTaskRunsCount)
.build();
}
}

View File

@@ -865,20 +865,18 @@ public class Execution implements DeletedInterface, TenantInterface {
* @param e the exception raise
* @return new taskRun with updated attempt with logs
*/
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun,
TaskRunAttempt lastAttempt, Exception e) {
private FailedTaskRunWithLog lastAttemptsTaskRunForFailedExecution(TaskRun taskRun, TaskRunAttempt lastAttempt, Exception e) {
TaskRun failed = taskRun
.withAttempts(
Stream
.concat(
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
Stream.of(lastAttempt.getState().isFailed() ? lastAttempt : lastAttempt.withState(State.Type.FAILED))
)
.toList()
);
return new FailedTaskRunWithLog(
taskRun
.withAttempts(
Stream
.concat(
taskRun.getAttempts().stream().limit(taskRun.getAttempts().size() - 1),
Stream.of(lastAttempt
.withState(State.Type.FAILED))
)
.toList()
)
.withState(State.Type.FAILED),
failed.getState().isFailed() ? failed : failed.withState(State.Type.FAILED),
RunContextLogger.logEntries(loggingEventFromException(e), LogEntry.of(taskRun, kind))
);
}

View File

@@ -62,6 +62,7 @@ public abstract class AbstractFlow implements FlowInterface {
@JsonSerialize(using = ListOrMapOfLabelSerializer.class)
@JsonDeserialize(using = ListOrMapOfLabelDeserializer.class)
@Schema(implementation = Object.class, oneOf = {List.class, Map.class})
@Valid
List<Label> labels;
@Schema(additionalProperties = Schema.AdditionalPropertiesValue.TRUE)

View File

@@ -185,34 +185,6 @@ public class Trigger extends TriggerContext implements HasUID {
.build();
}
public static Trigger update(Trigger currentTrigger, Trigger newTrigger, ZonedDateTime nextExecutionDate) throws Exception {
Trigger updated = currentTrigger;
// If a backfill is created, we update the currentTrigger
// and set the nextExecutionDate() as the previous one
if (newTrigger.getBackfill() != null) {
updated = currentTrigger.toBuilder()
.backfill(
newTrigger
.getBackfill()
.toBuilder()
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
.currentDate(
newTrigger.getBackfill().getStart()
)
.previousNextExecutionDate(
currentTrigger.getNextExecutionDate())
.build())
.build();
}
return updated.toBuilder()
.nextExecutionDate(newTrigger.getDisabled() ?
null : nextExecutionDate)
.disabled(newTrigger.getDisabled())
.build();
}
public Trigger resetExecution(Flow flow, Execution execution, ConditionContext conditionContext) {
boolean disabled = this.getStopAfter() != null ? this.getStopAfter().contains(execution.getState().getCurrent()) : this.getDisabled();
if (!disabled) {
@@ -276,27 +248,22 @@ public class Trigger extends TriggerContext implements HasUID {
.build();
}
public Trigger initBackfill(Trigger newTrigger) {
// If a backfill is created, we update the currentTrigger
public Trigger withBackfill(final Backfill backfill) {
Trigger updated = this;
// If a backfill is created, we update the trigger
// and set the nextExecutionDate() as the previous one
if (newTrigger.getBackfill() != null) {
return this.toBuilder()
if (backfill != null) {
updated = this.toBuilder()
.backfill(
newTrigger
.getBackfill()
backfill
.toBuilder()
.end(newTrigger.getBackfill().getEnd() != null ? newTrigger.getBackfill().getEnd() : ZonedDateTime.now())
.currentDate(
newTrigger.getBackfill().getStart()
)
.previousNextExecutionDate(
this.getNextExecutionDate())
.end(backfill.getEnd() != null ? backfill.getEnd() : ZonedDateTime.now())
.currentDate(backfill.getStart())
.previousNextExecutionDate(this.getNextExecutionDate())
.build())
.build();
}
return this;
return updated;
}
// if the next date is after the backfill end, we remove the backfill

View File

@@ -56,7 +56,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
*
* @return the {@link DefaultPluginRegistry}.
*/
public static DefaultPluginRegistry getOrCreate() {
public synchronized static DefaultPluginRegistry getOrCreate() {
DefaultPluginRegistry instance = LazyHolder.INSTANCE;
if (!instance.isInitialized()) {
instance.init();
@@ -74,7 +74,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
/**
* Initializes the registry by loading all core plugins.
*/
protected void init() {
protected synchronized void init() {
if (initialized.compareAndSet(false, true)) {
register(scanner.scan());
}
@@ -200,7 +200,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
if (existing != null && existing.crc32() == plugin.crc32()) {
return; // same plugin already registered
}
lock.lock();
try {
if (existing != null) {
@@ -212,7 +212,7 @@ public class DefaultPluginRegistry implements PluginRegistry {
lock.unlock();
}
}
protected void registerAll(Map<PluginIdentifier, PluginClassAndMetadata<? extends Plugin>> plugins) {
pluginClassByIdentifier.putAll(plugins);
}

View File

@@ -6,6 +6,12 @@ import lombok.Getter;
import lombok.ToString;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Enumeration;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.zip.CRC32;
@AllArgsConstructor
@Getter
@@ -14,5 +20,59 @@ import java.net.URL;
public class ExternalPlugin {
private final URL location;
private final URL[] resources;
private final long crc32;
private volatile Long crc32; // lazy-val
public ExternalPlugin(URL location, URL[] resources) {
this.location = location;
this.resources = resources;
}
public Long getCrc32() {
if (this.crc32 == null) {
synchronized (this) {
if (this.crc32 == null) {
this.crc32 = computeJarCrc32(location);
}
}
}
return crc32;
}
/**
* Compute a CRC32 of the JAR File without reading the whole file
*
* @param location of the JAR File.
* @return the CRC32 of {@code -1} if the checksum can't be computed.
*/
private static long computeJarCrc32(final URL location) {
CRC32 crc = new CRC32();
try (JarFile jar = new JarFile(location.toURI().getPath(), false)) {
Enumeration<JarEntry> entries = jar.entries();
byte[] buffer = new byte[Long.BYTES]; // reusable buffer to avoid re-allocation
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
crc.update(entry.getName().getBytes(StandardCharsets.UTF_8));
updateCrc32WithLong(crc, buffer, entry.getSize());
updateCrc32WithLong(crc, buffer, entry.getCrc());
}
return crc.getValue();
} catch (Exception e) {
return -1;
}
}
private static void updateCrc32WithLong(CRC32 crc32, byte[] reusable, long val) {
// fast long -> byte conversion
reusable[0] = (byte) (val >>> 56);
reusable[1] = (byte) (val >>> 48);
reusable[2] = (byte) (val >>> 40);
reusable[3] = (byte) (val >>> 32);
reusable[4] = (byte) (val >>> 24);
reusable[5] = (byte) (val >>> 16);
reusable[6] = (byte) (val >>> 8);
reusable[7] = (byte) val;
crc32.update(reusable);;
}
}

View File

@@ -46,6 +46,7 @@ public class PluginClassLoader extends URLClassLoader {
+ "|dev.failsafe"
+ "|reactor"
+ "|io.opentelemetry"
+ "|io.netty"
+ ")\\..*$");
private final ClassLoader parent;

View File

@@ -51,8 +51,7 @@ public class PluginResolver {
final List<URL> resources = resolveUrlsForPluginPath(path);
plugins.add(new ExternalPlugin(
path.toUri().toURL(),
resources.toArray(new URL[0]),
computeJarCrc32(path)
resources.toArray(new URL[0])
));
}
} catch (final InvalidPathException | MalformedURLException e) {
@@ -124,33 +123,5 @@ public class PluginResolver {
return urls;
}
/**
* Compute a CRC32 of the JAR File without reading the whole file
*
* @param location of the JAR File.
* @return the CRC32 of {@code -1} if the checksum can't be computed.
*/
private static long computeJarCrc32(final Path location) {
CRC32 crc = new CRC32();
try (JarFile jar = new JarFile(location.toFile(), false)) {
Enumeration<JarEntry> entries = jar.entries();
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
crc.update(entry.getName().getBytes());
crc.update(longToBytes(entry.getSize()));
crc.update(longToBytes(entry.getCrc()));
}
} catch (Exception e) {
return -1;
}
return crc.getValue();
}
private static byte[] longToBytes(long x) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(x);
return buffer.array();
}
}

View File

@@ -144,7 +144,7 @@ public final class PluginDeserializer<T extends Plugin> extends JsonDeserializer
static String extractPluginRawIdentifier(final JsonNode node, final boolean isVersioningSupported) {
String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null);
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null);
String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::asText).orElse(null);
if (type == null || type.isEmpty()) {
return null;

View File

@@ -25,8 +25,6 @@ import java.util.Optional;
import java.util.function.Function;
public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Execution>, QueryBuilderInterface<Executions.Fields> {
Boolean isTaskRunEnabled();
default Optional<Execution> findById(String tenantId, String id) {
return findById(tenantId, id, false);
}
@@ -96,12 +94,6 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
Flux<Execution> findAllAsync(@Nullable String tenantId);
ArrayListTotal<TaskRun> findTaskRun(
Pageable pageable,
@Nullable String tenantId,
List<QueryFilter> filters
);
Execution delete(Execution execution);
Integer purge(Execution execution);
@@ -112,8 +104,7 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
@Nullable String flowId,
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
boolean isTaskRun
@Nullable DateUtils.GroupType groupBy
);
List<DailyExecutionStatistics> dailyStatistics(
@@ -125,8 +116,7 @@ public interface ExecutionRepositoryInterface extends SaveRepositoryInterface<Ex
@Nullable ZonedDateTime startDate,
@Nullable ZonedDateTime endDate,
@Nullable DateUtils.GroupType groupBy,
List<State.Type> state,
boolean isTaskRun
List<State.Type> state
);
@Getter

View File

@@ -83,7 +83,9 @@ public class LocalFlowRepositoryLoader {
}
public void load(String tenantId, File basePath) throws IOException {
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants().stream()
Map<String, FlowInterface> flowByUidInRepository = flowRepository.findAllForAllTenants()
.stream()
.filter(flow -> tenantId.equals(flow.getTenantId()))
.collect(Collectors.toMap(FlowId::uidWithoutRevision, Function.identity()));
try (Stream<Path> pathStream = Files.walk(basePath.toPath())) {

View File

@@ -5,10 +5,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.models.Label;
import io.kestra.core.models.executions.*;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.flows.FlowWithException;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.flows.*;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.tasks.ExecutableTask;
import io.kestra.core.models.tasks.Task;
@@ -29,6 +26,7 @@ import org.apache.commons.lang3.stream.Streams;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
import static io.kestra.core.trace.Tracer.throwCallable;
import static io.kestra.core.utils.Rethrow.throwConsumer;
@@ -153,17 +151,24 @@ public final class ExecutableUtils {
currentFlow.getNamespace(),
currentFlow.getId()
)
.orElseThrow(() -> new IllegalStateException("Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'"));
.orElseThrow(() -> {
String msg = "Unable to find flow '" + subflowNamespace + "'.'" + subflowId + "' with revision '" + subflowRevision.orElse(0) + "'";
runContext.logger().error(msg);
return new IllegalStateException(msg);
});
if (flow.isDisabled()) {
throw new IllegalStateException("Cannot execute a flow which is disabled");
String msg = "Cannot execute a flow which is disabled";
runContext.logger().error(msg);
throw new IllegalStateException(msg);
}
if (flow instanceof FlowWithException fwe) {
throw new IllegalStateException("Cannot execute an invalid flow: " + fwe.getException());
String msg = "Cannot execute an invalid flow: " + fwe.getException();
runContext.logger().error(msg);
throw new IllegalStateException(msg);
}
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
List<Label> newLabels = inheritLabels ? new ArrayList<>(filterLabels(currentExecution.getLabels(), flow)) : new ArrayList<>(systemLabels(currentExecution));
if (labels != null) {
labels.forEach(throwConsumer(label -> newLabels.add(new Label(runContext.render(label.key()), runContext.render(label.value())))));
}
@@ -201,7 +206,20 @@ public final class ExecutableUtils {
.build()
)
.withScheduleDate(scheduleOnDate);
if(execution.getInputs().size()<inputs.size()) {
Map<String,Object>resolvedInputs=execution.getInputs();
for (var inputKey : inputs.keySet()) {
if (!resolvedInputs.containsKey(inputKey)) {
runContext.logger().warn(
"Input {} was provided by parent execution {} for subflow {}.{} but isn't declared at the subflow inputs",
inputKey,
currentExecution.getId(),
currentTask.subflowId().namespace(),
currentTask.subflowId().flowId()
);
}
}
}
// inject the traceparent into the new execution
propagator.ifPresent(pg -> pg.inject(Context.current(), execution, ExecutionTextMapSetter.INSTANCE));

View File

@@ -49,15 +49,7 @@ import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.*;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Matcher;
@@ -231,6 +223,19 @@ public class FlowInputOutput {
return new AbstractMap.SimpleEntry<>(it.input().getId(), it.value());
})
.collect(HashMap::new, (m,v)-> m.put(v.getKey(), v.getValue()), HashMap::putAll);
if (resolved.size() < data.size()) {
RunContext runContext = runContextFactory.of(flow, execution);
for (var inputKey : data.keySet()) {
if (!resolved.containsKey(inputKey)) {
runContext.logger().warn(
"Input {} was provided for workflow {}.{} but isn't declared in the workflow inputs",
inputKey,
flow.getNamespace(),
flow.getId()
);
}
}
}
return MapUtils.flattenToNestedMap(resolved);
}
@@ -313,15 +318,15 @@ public class FlowInputOutput {
});
resolvable.setInput(input);
Object value = resolvable.get().value();
// resolve default if needed
if (value == null && input.getDefaults() != null) {
value = resolveDefaultValue(input, runContext);
resolvable.isDefault(true);
}
// validate and parse input value
if (value == null) {
if (input.getRequired()) {
@@ -350,7 +355,7 @@ public class FlowInputOutput {
return resolvable.get();
}
public static Object resolveDefaultValue(Input<?> input, PropertyContext renderer) throws IllegalVariableEvaluationException {
return switch (input.getType()) {
case STRING, ENUM, SELECT, SECRET, EMAIL -> resolveDefaultPropertyAs(input, renderer, String.class);
@@ -367,7 +372,7 @@ public class FlowInputOutput {
case MULTISELECT -> resolveDefaultPropertyAsList(input, renderer, String.class);
};
}
@SuppressWarnings("unchecked")
private static <T> Object resolveDefaultPropertyAs(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
return Property.as((Property<T>) input.getDefaults(), renderer, clazz);
@@ -376,7 +381,7 @@ public class FlowInputOutput {
private static <T> Object resolveDefaultPropertyAsList(Input<?> input, PropertyContext renderer, Class<T> clazz) throws IllegalVariableEvaluationException {
return Property.asList((Property<List<T>>) input.getDefaults(), renderer, clazz);
}
private RunContext buildRunContextForExecutionAndInputs(final FlowInterface flow, final Execution execution, Map<String, InputAndValue> dependencies) {
Map<String, Object> flattenInputs = MapUtils.flattenToNestedMap(dependencies.entrySet()
.stream()
@@ -453,7 +458,7 @@ public class FlowInputOutput {
if (data.getType() == null) {
return Optional.of(new AbstractMap.SimpleEntry<>(data.getId(), current));
}
final Type elementType = data instanceof ItemTypeInterface itemTypeInterface ? itemTypeInterface.getItemType() : null;
return Optional.of(new AbstractMap.SimpleEntry<>(
@@ -530,17 +535,17 @@ public class FlowInputOutput {
throw new Exception("Expected `" + type + "` but received `" + current + "` with errors:\n```\n" + e.getMessage() + "\n```");
}
}
public static Map<String, Object> renderFlowOutputs(List<Output> outputs, RunContext runContext) throws IllegalVariableEvaluationException {
if (outputs == null) return Map.of();
// render required outputs
Map<String, Object> outputsById = outputs
.stream()
.filter(output -> output.getRequired() == null || output.getRequired())
.collect(HashMap::new, (map, entry) -> map.put(entry.getId(), entry.getValue()), Map::putAll);
outputsById = runContext.render(outputsById);
// render optional outputs one by one to catch, log, and skip any error.
for (io.kestra.core.models.flows.Output output : outputs) {
if (Boolean.FALSE.equals(output.getRequired())) {
@@ -583,9 +588,9 @@ public class FlowInputOutput {
}
public void isDefault(boolean isDefault) {
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
this.input = new InputAndValue(this.input.input(), this.input.value(), this.input.enabled(), isDefault, this.input.exception());
}
public void setInput(final Input<?> input) {
this.input = new InputAndValue(input, this.input.value(), this.input.enabled(), this.input.isDefault(), this.input.exception());
}

View File

@@ -500,7 +500,7 @@ public class FlowableUtils {
ArrayList<ResolvedTask> result = new ArrayList<>();
int index = 0;
int iteration = 0;
for (Object current : distinctValue) {
try {
String resolvedValue = current instanceof String stringValue ? stringValue : MAPPER.writeValueAsString(current);
@@ -508,7 +508,7 @@ public class FlowableUtils {
result.add(ResolvedTask.builder()
.task(task)
.value(resolvedValue)
.iteration(index++)
.iteration(iteration)
.parentId(parentTaskRun.getId())
.build()
);
@@ -516,6 +516,7 @@ public class FlowableUtils {
} catch (JsonProcessingException e) {
throw new IllegalVariableEvaluationException(e);
}
iteration++;
}
return result;

View File

@@ -10,6 +10,7 @@ import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.flows.Input;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.flows.input.SecretInput;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.property.PropertyContext;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -282,15 +283,15 @@ public final class RunVariables {
if (flow != null && flow.getInputs() != null) {
// we add default inputs value from the flow if not already set, this will be useful for triggers
flow.getInputs().stream()
.filter(input -> input.getDefaults() != null && !inputs.containsKey(input.getId()))
.forEach(input -> {
try {
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, propertyContext));
} catch (IllegalVariableEvaluationException e) {
throw new RuntimeException("Unable to inject default value for input '" + input.getId() + "'", e);
}
});
flow.getInputs().stream()
.filter(input -> input.getDefaults() != null && !inputs.containsKey(input.getId()))
.forEach(input -> {
try {
inputs.put(input.getId(), FlowInputOutput.resolveDefaultValue(input, propertyContext));
} catch (IllegalVariableEvaluationException e) {
// Silent catch, if an input depends on another input, or a variable that is populated at runtime / input filling time, we can't resolve it here.
}
});
}
if (!inputs.isEmpty()) {

View File

@@ -45,7 +45,7 @@ final class Secret {
for (var entry: data.entrySet()) {
if (entry.getValue() instanceof Map map) {
// if some value are of type EncryptedString we decode them and replace the object
if (EncryptedString.TYPE.equalsIgnoreCase((String)map.get("type"))) {
if (map.get("type") instanceof String typeStr && EncryptedString.TYPE.equalsIgnoreCase(typeStr)) {
try {
String decoded = decrypt((String) map.get("value"));
decryptedMap.put(entry.getKey(), decoded);

View File

@@ -168,6 +168,7 @@ public class Extension extends AbstractExtension {
functions.put("randomPort", new RandomPortFunction());
functions.put("fileExists", fileExistsFunction);
functions.put("isFileEmpty", isFileEmptyFunction);
functions.put("nanoId", new NanoIDFunction());
functions.put("tasksWithState", new TasksWithStateFunction());
functions.put(HttpFunction.NAME, httpFunction);
return functions;

View File

@@ -30,6 +30,6 @@ public class TimestampMicroFilter extends AbstractDate implements Filter {
ZoneId zoneId = zoneId(timeZone);
ZonedDateTime date = convert(input, zoneId, existingFormat);
return String.valueOf(TimeUnit.SECONDS.toNanos(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
return String.valueOf(TimeUnit.SECONDS.toMicros(date.toEpochSecond()) + TimeUnit.NANOSECONDS.toMicros(date.getNano()));
}
}

View File

@@ -5,6 +5,8 @@ import io.kestra.core.http.HttpRequest;
import io.kestra.core.http.HttpResponse;
import io.kestra.core.http.client.HttpClient;
import io.kestra.core.http.client.HttpClientException;
import io.kestra.core.http.client.HttpClientRequestException;
import io.kestra.core.http.client.HttpClientResponseException;
import io.kestra.core.http.client.configurations.HttpConfiguration;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
@@ -101,8 +103,15 @@ public class HttpFunction<T> implements Function {
try (HttpClient httpClient = new HttpClient(runContext, httpConfiguration)) {
HttpResponse<Object> response = httpClient.request(httpRequest, Object.class);
return response.getBody();
} catch (HttpClientException | IllegalVariableEvaluationException | IOException e) {
throw new PebbleException(e, "Unable to execute HTTP request", lineNumber, self.getName());
} catch (HttpClientResponseException e) {
if (e.getResponse() != null) {
String msg = "Failed to execute HTTP Request, server respond with status " + e.getResponse().getStatus().getCode() + " : " + e.getResponse().getStatus().getReason();
throw new PebbleException(e, msg , lineNumber, self.getName());
} else {
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
}
} catch(HttpClientException | IllegalVariableEvaluationException | IOException e ) {
throw new PebbleException( e, "Failed to execute HTTP request ", lineNumber, self.getName());
}
}

View File

@@ -0,0 +1,66 @@
package io.kestra.core.runners.pebble.functions;
import io.pebbletemplates.pebble.error.PebbleException;
import io.pebbletemplates.pebble.extension.Function;
import io.pebbletemplates.pebble.template.EvaluationContext;
import io.pebbletemplates.pebble.template.PebbleTemplate;
import java.security.SecureRandom;
import java.util.List;
import java.util.Map;
public class NanoIDFunction implements Function {
private static final int DEFAULT_LENGTH = 21;
private static final char[] DEFAULT_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".toCharArray();
private static final SecureRandom secureRandom = new SecureRandom();
private static final String LENGTH = "length";
private static final String ALPHABET = "alphabet";
private static final int MAX_LENGTH = 1000;
@Override
public Object execute(
Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
int length = DEFAULT_LENGTH;
if (args.containsKey(LENGTH) && (args.get(LENGTH) instanceof Long)) {
length = parseLength(args, self, lineNumber);
}
char[] alphabet = DEFAULT_ALPHABET;
if (args.containsKey(ALPHABET) && (args.get(ALPHABET) instanceof String)) {
alphabet = ((String) args.get(ALPHABET)).toCharArray();
}
return createNanoID(length, alphabet);
}
private static int parseLength(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
var value = (Long) args.get(LENGTH);
if(value > MAX_LENGTH) {
throw new PebbleException(
null,
"The 'nanoId()' function field 'length' must be lower than: " + MAX_LENGTH,
lineNumber,
self.getName());
}
return Math.toIntExact(value);
}
@Override
public List<String> getArgumentNames() {
return List.of(LENGTH,ALPHABET);
}
String createNanoID(int length, char[] alphabet){
final char[] data = new char[length];
final byte[] bytes = new byte[length];
final int mask = alphabet.length-1;
secureRandom.nextBytes(bytes);
for (int i = 0; i < length; ++i) {
data[i] = alphabet[bytes[i] & mask];
}
return String.valueOf(data);
}
}

View File

@@ -180,23 +180,13 @@ public final class FileSerde {
}
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, TypeReference<T> type) throws IOException {
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
// and a sequence/stream of things (of Array shape).
// To work around that, we need to create a JsonParser and advance to the first token.
try (var parser = objectMapper.createParser(reader)) {
parser.nextToken();
return objectMapper.readerFor(type).readValues(parser);
}
}
private static <T> MappingIterator<T> createMappingIterator(ObjectMapper objectMapper, Reader reader, Class<T> type) throws IOException {
// See https://github.com/FasterXML/jackson-dataformats-binary/issues/493
// There is a limitation with the MappingIterator that cannot differentiate between an array of things (of whatever shape)
// and a sequence/stream of things (of Array shape).
// To work around that, we need to create a JsonParser and advance to the first token.
try (var parser = objectMapper.createParser(reader)) {
parser.nextToken();
return objectMapper.readerFor(type).readValues(parser);
}
}

View File

@@ -163,31 +163,28 @@ public final class JacksonMapper {
.build();
}
public static Pair<JsonNode, JsonNode> getBiDirectionalDiffs(Object previous, Object current) {
JsonNode previousJson = MAPPER.valueToTree(previous);
JsonNode newJson = MAPPER.valueToTree(current);
public static Pair<JsonNode, JsonNode> getBiDirectionalDiffs(Object before, Object after) {
JsonNode beforeNode = MAPPER.valueToTree(before);
JsonNode afterNode = MAPPER.valueToTree(after);
JsonNode patchPrevToNew = JsonDiff.asJson(previousJson, newJson);
JsonNode patchNewToPrev = JsonDiff.asJson(newJson, previousJson);
JsonNode patch = JsonDiff.asJson(beforeNode, afterNode);
JsonNode revert = JsonDiff.asJson(afterNode, beforeNode);
return Pair.of(patchPrevToNew, patchNewToPrev);
return Pair.of(patch, revert);
}
public static String applyPatches(Object object, List<JsonNode> patches) throws JsonProcessingException {
public static JsonNode applyPatchesOnJsonNode(JsonNode jsonObject, List<JsonNode> patches) {
for (JsonNode patch : patches) {
try {
// Required for ES
if (patch.findValue("value") == null) {
((ObjectNode) patch.get(0)).set("value", (JsonNode) null);
if (patch.findValue("value") == null && !patch.isEmpty()) {
((ObjectNode) patch.get(0)).set("value", null);
}
JsonNode current = MAPPER.valueToTree(object);
object = JsonPatch.fromJson(patch).apply(current);
jsonObject = JsonPatch.fromJson(patch).apply(jsonObject);
} catch (IOException | JsonPatchException e) {
throw new RuntimeException(e);
}
}
return MAPPER.writeValueAsString(object);
return jsonObject;
}
}

View File

@@ -3,12 +3,7 @@ package io.kestra.core.services;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.kestra.core.exceptions.FlowProcessingException;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowId;
import io.kestra.core.models.flows.FlowInterface;
import io.kestra.core.models.flows.FlowWithException;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.flows.GenericFlow;
import io.kestra.core.models.flows.*;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.models.topologies.FlowTopology;
import io.kestra.core.models.triggers.AbstractTrigger;
@@ -30,16 +25,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -551,23 +537,24 @@ public class FlowService {
return expandAll ? recursiveFlowTopology(new ArrayList<>(), tenant, namespace, id, destinationOnly) : flowTopologyRepository.get().findByFlow(tenant, namespace, id, destinationOnly).stream();
}
private Stream<FlowTopology> recursiveFlowTopology(List<FlowId> flowIds, String tenantId, String namespace, String id, boolean destinationOnly) {
private Stream<FlowTopology> recursiveFlowTopology(List<String> visitedTopologies, String tenantId, String namespace, String id, boolean destinationOnly) {
if (flowTopologyRepository.isEmpty()) {
throw noRepositoryException();
}
List<FlowTopology> flowTopologies = flowTopologyRepository.get().findByFlow(tenantId, namespace, id, destinationOnly);
FlowId flowId = FlowId.of(tenantId, namespace, id, null);
if (flowIds.contains(flowId)) {
return flowTopologies.stream();
}
flowIds.add(flowId);
var flowTopologies = flowTopologyRepository.get().findByFlow(tenantId, namespace, id, destinationOnly);
return flowTopologies.stream()
.flatMap(topology -> Stream.of(topology.getDestination(), topology.getSource()))
// recursively fetch child nodes
.flatMap(node -> recursiveFlowTopology(flowIds, node.getTenantId(), node.getNamespace(), node.getId(), destinationOnly));
// ignore already visited topologies
.filter(x -> !visitedTopologies.contains(x.uid()))
.flatMap(topology -> {
visitedTopologies.add(topology.uid());
Stream<FlowTopology> subTopologies = Stream
.of(topology.getDestination(), topology.getSource())
// recursively visit children and parents nodes
.flatMap(relationNode -> recursiveFlowTopology(visitedTopologies, relationNode.getTenantId(), relationNode.getNamespace(), relationNode.getId(), destinationOnly));
return Stream.concat(Stream.of(topology), subTopologies);
});
}
private IllegalStateException noRepositoryException() {

View File

@@ -1,5 +1,6 @@
package io.kestra.core.storages.kv;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import java.time.Duration;
@@ -9,6 +10,7 @@ import java.util.Map;
import java.util.Optional;
@Getter
@EqualsAndHashCode
public class KVMetadata {
private String description;
private Instant expirationDate;
@@ -17,14 +19,18 @@ public class KVMetadata {
if (ttl != null && ttl.isNegative()) {
throw new IllegalArgumentException("ttl cannot be negative");
}
this.description = description;
if (ttl != null) {
this.expirationDate = Instant.now().plus(ttl);
}
}
public KVMetadata(String description, Instant expirationDate) {
this.description = description;
this.expirationDate = expirationDate;
}
public KVMetadata(Map<String, String> metadata) {
if (metadata == null) {
return;
@@ -46,4 +52,9 @@ public class KVMetadata {
}
return map;
}
@Override
public String toString() {
return "[description=" + description + ", expirationDate=" + expirationDate + "]";
}
}

View File

@@ -4,7 +4,9 @@ import io.kestra.core.exceptions.ResourceExpiredException;
import io.kestra.core.storages.StorageContext;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern;
@@ -104,8 +106,33 @@ public interface KVStore {
default boolean exists(String key) throws IOException {
return list().stream().anyMatch(kvEntry -> kvEntry.key().equals(key));
}
/**
* Finds a KV entry with associated metadata for a given key.
*
* @param key the KV entry key.
* @return an optional of {@link KVValueAndMetadata}.
*
* @throws UncheckedIOException if an error occurred while executing the operation on the K/V store.
*/
default Optional<KVValueAndMetadata> findMetadataAndValue(final String key) throws UncheckedIOException {
try {
return get(key).flatMap(entry ->
{
try {
return getValue(entry.key()).map(current -> new KVValueAndMetadata(new KVMetadata(entry.description(), entry.expirationDate()), current.value()));
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (ResourceExpiredException e) {
return Optional.empty();
}
}
);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
Pattern KEY_VALIDATOR_PATTERN = Pattern.compile("[a-zA-Z0-9][a-zA-Z0-9._-]*");
/**

View File

@@ -18,6 +18,7 @@ import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.repositories.FlowTopologyRepositoryInterface;
import io.kestra.core.services.ConditionService;
import io.kestra.core.utils.ListUtils;
import io.kestra.core.utils.MapUtils;
import io.kestra.plugin.core.condition.*;
import io.micronaut.core.annotation.Nullable;
import jakarta.inject.Inject;
@@ -175,9 +176,6 @@ public class FlowTopologyService {
protected boolean isTriggerChild(Flow parent, Flow child) {
List<AbstractTrigger> triggers = ListUtils.emptyOnNull(child.getTriggers());
// simulated execution: we add a "simulated" label so conditions can know that the evaluation is for a simulated execution
Execution execution = Execution.newExecution(parent, (f, e) -> null, List.of(SIMULATED_EXECUTION), Optional.empty());
// keep only flow trigger
List<io.kestra.plugin.core.trigger.Flow> flowTriggers = triggers
.stream()
@@ -189,13 +187,16 @@ public class FlowTopologyService {
return false;
}
// simulated execution: we add a "simulated" label so conditions can know that the evaluation is for a simulated execution
Execution execution = Execution.newExecution(parent, (f, e) -> null, List.of(SIMULATED_EXECUTION), Optional.empty());
boolean conditionMatch = flowTriggers
.stream()
.flatMap(flow -> ListUtils.emptyOnNull(flow.getConditions()).stream())
.allMatch(condition -> validateCondition(condition, parent, execution));
boolean preconditionMatch = flowTriggers.stream()
.anyMatch(flow -> flow.getPreconditions() == null || validateMultipleConditions(flow.getPreconditions().getConditions(), parent, execution));
.anyMatch(flow -> flow.getPreconditions() == null || validatePreconditions(flow.getPreconditions(), parent, execution));
return conditionMatch && preconditionMatch;
}
@@ -239,11 +240,24 @@ public class FlowTopologyService {
}
private boolean isMandatoryMultipleCondition(Condition condition) {
return Stream
.of(
Expression.class
)
.anyMatch(aClass -> condition.getClass().isAssignableFrom(aClass));
return condition.getClass().isAssignableFrom(Expression.class);
}
private boolean validatePreconditions(io.kestra.plugin.core.trigger.Flow.Preconditions preconditions, FlowInterface child, Execution execution) {
boolean upstreamFlowMatched = MapUtils.emptyOnNull(preconditions.getUpstreamFlowsConditions())
.values()
.stream()
.filter(c -> !isFilterCondition(c))
.anyMatch(c -> validateCondition(c, child, execution));
boolean whereMatched = MapUtils.emptyOnNull(preconditions.getWhereConditions())
.values()
.stream()
.filter(c -> !isFilterCondition(c))
.allMatch(c -> validateCondition(c, child, execution));
// to be a dependency, if upstream flow is set it must be either inside it so it's a AND between upstream flow and where
return upstreamFlowMatched && whereMatched;
}
private boolean isFilterCondition(Condition condition) {

View File

@@ -206,22 +206,17 @@ public class MapUtils {
/**
* Utility method that flatten a nested map.
* <p>
* NOTE: for simplicity, this method didn't allow to flatten maps with conflicting keys that would end up in different flatten keys,
* this could be related later if needed by flattening {k1: k2: {k3: v1}, k1: {k4: v2}} to {k1.k2.k3: v1, k1.k4: v2} is prohibited for now.
*
* @param nestedMap the nested map.
* @return the flattened map.
*
* @throws IllegalArgumentException if any entry contains a map of more than one element.
*/
public static Map<String, Object> nestedToFlattenMap(@NotNull Map<String, Object> nestedMap) {
Map<String, Object> result = new TreeMap<>();
for (Map.Entry<String, Object> entry : nestedMap.entrySet()) {
if (entry.getValue() instanceof Map<?, ?> map) {
Map.Entry<String, Object> flatten = flattenEntry(entry.getKey(), (Map<String, Object>) map);
result.put(flatten.getKey(), flatten.getValue());
Map<String, Object> flatten = flattenEntry(entry.getKey(), (Map<String, Object>) map);
result.putAll(flatten);
} else {
result.put(entry.getKey(), entry.getValue());
}
@@ -229,18 +224,19 @@ public class MapUtils {
return result;
}
private static Map.Entry<String, Object> flattenEntry(String key, Map<String, Object> value) {
if (value.size() > 1) {
throw new IllegalArgumentException("You cannot flatten a map with an entry that is a map of more than one element, conflicting key: " + key);
private static Map<String, Object> flattenEntry(String key, Map<String, Object> value) {
Map<String, Object> result = new TreeMap<>();
for (Map.Entry<String, Object> entry : value.entrySet()) {
String newKey = key + "." + entry.getKey();
Object newValue = entry.getValue();
if (newValue instanceof Map<?, ?> map) {
result.putAll(flattenEntry(newKey, (Map<String, Object>) map));
} else {
result.put(newKey, newValue);
}
}
Map.Entry<String, Object> entry = value.entrySet().iterator().next();
String newKey = key + "." + entry.getKey();
Object newValue = entry.getValue();
if (newValue instanceof Map<?, ?> map) {
return flattenEntry(newKey, (Map<String, Object>) map);
} else {
return Map.entry(newKey, newValue);
}
return result;
}
}

View File

@@ -32,48 +32,84 @@ public class Version implements Comparable<Version> {
* @param version the version.
* @return a new {@link Version} instance.
*/
public static Version of(String version) {
public static Version of(final Object version) {
if (version.startsWith("v")) {
version = version.substring(1);
if (Objects.isNull(version)) {
throw new IllegalArgumentException("Invalid version, cannot parse null version");
}
String strVersion = version.toString();
if (strVersion.startsWith("v")) {
strVersion = strVersion.substring(1);
}
int qualifier = version.indexOf("-");
int qualifier = strVersion.indexOf("-");
final String[] versions = qualifier > 0 ?
version.substring(0, qualifier).split("\\.") :
version.split("\\.");
strVersion.substring(0, qualifier).split("\\.") :
strVersion.split("\\.");
try {
final int majorVersion = Integer.parseInt(versions[0]);
final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0;
final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0;
final Integer minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : null;
final Integer incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : null;
return new Version(
majorVersion,
minorVersion,
incrementalVersion,
qualifier > 0 ? version.substring(qualifier + 1) : null,
version
qualifier > 0 ? strVersion.substring(qualifier + 1) : null,
strVersion
);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'");
}
}
/**
* Static helper method for returning the most recent stable version for a current {@link Version}.
* Resolves the most appropriate stable version from a collection, based on a given input version.
* <p>
* The matching rules are:
* <ul>
* <li>If {@code from} specifies only a major version (e.g. {@code 1}), return the latest stable version
* with the same major (e.g. {@code 1.2.3}).</li>
* <li>If {@code from} specifies a major and minor version only (e.g. {@code 1.2}), return the latest
* stable version with the same major and minor (e.g. {@code 1.2.3}).</li>
* <li>If {@code from} specifies a full version with major, minor, and patch (e.g. {@code 1.2.2}),
* then only return it if it is exactly present (and stable) in {@code versions}.
* No "upgrade" is performed in this case.</li>
* <li>If no suitable version is found, returns {@code null}.</li>
* </ul>
*
* @param from the current version.
* @param versions the list of version.
*
* @return the last stable version.
* @param from the reference version (may specify only major, or major+minor, or major+minor+patch).
* @param versions the collection of candidate versions to resolve against.
* @return the best matching stable version, or {@code null} if none match.
*/
public static Version getStable(final Version from, final Collection<Version> versions) {
List<Version> compatibleVersions = versions.stream()
.filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion())
.toList();
if (compatibleVersions.isEmpty()) return null;
return Version.getLatest(compatibleVersions);
// Case 1: "from" is only a major (e.g. 1)
if (from.hasOnlyMajor()) {
List<Version> sameMajor = versions.stream()
.filter(v -> v.majorVersion() == from.majorVersion())
.toList();
return sameMajor.isEmpty() ? null : Version.getLatest(sameMajor);
}
// Case 2: "from" is major+minor only (e.g. 1.2)
if (from.hasMajorAndMinorOnly()) {
List<Version> sameMinor = versions.stream()
.filter(v -> v.majorVersion() == from.majorVersion()
&& v.minorVersion() == from.minorVersion())
.toList();
return sameMinor.isEmpty() ? null : Version.getLatest(sameMinor);
}
// Case 3: "from" is full version (major+minor+patch)
if (versions.contains(from)) {
return from;
}
// No match
return null;
}
/**
@@ -123,8 +159,8 @@ public class Version implements Comparable<Version> {
}
private final int majorVersion;
private final int minorVersion;
private final int incrementalVersion;
private final Integer minorVersion;
private final Integer patchVersion;
private final Qualifier qualifier;
private final String originalVersion;
@@ -134,14 +170,14 @@ public class Version implements Comparable<Version> {
*
* @param majorVersion the major version (must be superior or equal to 0).
* @param minorVersion the minor version (must be superior or equal to 0).
* @param incrementalVersion the incremental version (must be superior or equal to 0).
* @param patchVersion the incremental version (must be superior or equal to 0).
* @param qualifier the qualifier.
*/
public Version(final int majorVersion,
final int minorVersion,
final int incrementalVersion,
final int patchVersion,
final String qualifier) {
this(majorVersion, minorVersion, incrementalVersion, qualifier, null);
this(majorVersion, minorVersion, patchVersion, qualifier, null);
}
/**
@@ -149,25 +185,25 @@ public class Version implements Comparable<Version> {
*
* @param majorVersion the major version (must be superior or equal to 0).
* @param minorVersion the minor version (must be superior or equal to 0).
* @param incrementalVersion the incremental version (must be superior or equal to 0).
* @param patchVersion the incremental version (must be superior or equal to 0).
* @param qualifier the qualifier.
* @param originalVersion the original string version.
*/
private Version(final int majorVersion,
final int minorVersion,
final int incrementalVersion,
private Version(final Integer majorVersion,
final Integer minorVersion,
final Integer patchVersion,
final String qualifier,
final String originalVersion) {
this.majorVersion = requirePositive(majorVersion, "major");
this.minorVersion = requirePositive(minorVersion, "minor");
this.incrementalVersion = requirePositive(incrementalVersion, "incremental");
this.patchVersion = requirePositive(patchVersion, "incremental");
this.qualifier = qualifier != null ? new Qualifier(qualifier) : null;
this.originalVersion = originalVersion;
}
private static int requirePositive(int version, final String message) {
if (version < 0) {
private static Integer requirePositive(Integer version, final String message) {
if (version != null && version < 0) {
throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message));
}
return version;
@@ -178,11 +214,11 @@ public class Version implements Comparable<Version> {
}
public int minorVersion() {
return minorVersion;
return minorVersion != null ? minorVersion : 0;
}
public int incrementalVersion() {
return incrementalVersion;
public int patchVersion() {
return patchVersion != null ? patchVersion : 0;
}
public Qualifier qualifier() {
@@ -197,9 +233,9 @@ public class Version implements Comparable<Version> {
if (this == o) return true;
if (!(o instanceof Version)) return false;
Version version = (Version) o;
return majorVersion == version.majorVersion &&
minorVersion == version.minorVersion &&
incrementalVersion == version.incrementalVersion &&
return Objects.equals(majorVersion,version.majorVersion) &&
Objects.equals(minorVersion, version.minorVersion) &&
Objects.equals(patchVersion,version.patchVersion) &&
Objects.equals(qualifier, version.qualifier);
}
@@ -208,7 +244,7 @@ public class Version implements Comparable<Version> {
*/
@Override
public int hashCode() {
return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier);
return Objects.hash(majorVersion, minorVersion, patchVersion, qualifier);
}
/**
@@ -218,7 +254,7 @@ public class Version implements Comparable<Version> {
public String toString() {
if (originalVersion != null) return originalVersion;
String version = majorVersion + "." + minorVersion + "." + incrementalVersion;
String version = majorVersion + "." + minorVersion + "." + patchVersion;
return (qualifier != null) ? version +"-" + qualifier : version;
}
@@ -238,7 +274,7 @@ public class Version implements Comparable<Version> {
return compareMinor;
}
int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion);
int compareIncremental = Integer.compare(that.patchVersion, this.patchVersion);
if (compareIncremental != 0) {
return compareIncremental;
}
@@ -253,6 +289,21 @@ public class Version implements Comparable<Version> {
return this.qualifier.compareTo(that.qualifier);
}
/**
* @return true if only major is specified (e.g. "1")
*/
private boolean hasOnlyMajor() {
return minorVersion == null && patchVersion == null;
}
/**
* @return true if major+minor are specified, but no patch (e.g. "1.2")
*/
private boolean hasMajorAndMinorOnly() {
return minorVersion != null && patchVersion == null;
}
/**
* Checks whether this version is before the given one.

View File

@@ -46,16 +46,19 @@ public class VersionProvider {
this.date = loadTime(gitProperties);
this.version = loadVersion(buildProperties, gitProperties);
// check the version in the settings and update if needed, we did't use it would allow us to detect incompatible update later if needed
if (settingRepository.isPresent()) {
Optional<Setting> versionSetting = settingRepository.get().findByKey(Setting.INSTANCE_VERSION);
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(this.version)) {
settingRepository.get().save(Setting.builder()
.key(Setting.INSTANCE_VERSION)
.value(this.version)
.build()
);
}
// check the version in the settings and update if needed, we didn't use it would allow us to detect incompatible update later if needed
settingRepository.ifPresent(
settingRepositoryInterface -> persistVersion(settingRepositoryInterface, version));
}
private static synchronized void persistVersion(SettingRepositoryInterface settingRepositoryInterface, String version) {
Optional<Setting> versionSetting = settingRepositoryInterface.findByKey(Setting.INSTANCE_VERSION);
if (versionSetting.isEmpty() || !versionSetting.get().getValue().equals(version)) {
settingRepositoryInterface.save(Setting.builder()
.key(Setting.INSTANCE_VERSION)
.value(version)
.build()
);
}
}

View File

@@ -21,6 +21,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.*;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
import java.util.Optional;
@@ -68,6 +69,7 @@ import java.util.Optional;
)
}
)
@Slf4j
public class Exit extends Task implements ExecutionUpdatableTask {
@NotNull
@Schema(
@@ -104,12 +106,13 @@ public class Exit extends Task implements ExecutionUpdatableTask {
// ends all parents
while (newTaskRun.getParentTaskRunId() != null) {
newTaskRun = newExecution.findTaskRunByTaskRunId(newTaskRun.getParentTaskRunId()).withState(exitState);
newExecution = execution.withTaskRun(newTaskRun);
newExecution = newExecution.withTaskRun(newTaskRun);
}
return newExecution;
} catch (InternalException e) {
// in case we cannot update the last not terminated task run, we ignore it
return execution;
log.warn("Unable to update the taskrun state", e);
return execution.withState(exitState);
}
})
.orElse(execution)

View File

@@ -127,9 +127,24 @@ public class Labels extends Task implements ExecutionUpdatableTask {
}
// check for system labels: none can be passed at runtime
Optional<Map.Entry<String, String>> first = labelsAsMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX)).findFirst();
if (first.isPresent()) {
throw new IllegalArgumentException("System labels can only be set by Kestra itself, offending label: " + first.get().getKey() + "=" + first.get().getValue());
Optional<Map.Entry<String, String>> systemLabel = labelsAsMap.entrySet().stream()
.filter(entry -> entry.getKey().startsWith(SYSTEM_PREFIX))
.findFirst();
if (systemLabel.isPresent()) {
throw new IllegalArgumentException(
"System labels can only be set by Kestra itself, offending label: " +
systemLabel.get().getKey() + "=" + systemLabel.get().getValue()
);
}
// check for empty label values
Optional<Map.Entry<String, String>> emptyValue = labelsAsMap.entrySet().stream()
.filter(entry -> entry.getValue().isEmpty())
.findFirst();
if (emptyValue.isPresent()) {
throw new IllegalArgumentException(
"Label values cannot be empty, offending label: " + emptyValue.get().getKey()
);
}
Map<String, String> newLabels = ListUtils.emptyOnNull(execution.getLabels()).stream()
@@ -140,6 +155,7 @@ public class Labels extends Task implements ExecutionUpdatableTask {
newLabels.putAll(labelsAsMap);
return execution.withLabels(newLabels.entrySet().stream()
.filter(Label.getEntryNotEmptyPredicate())
.map(entry -> new Label(
entry.getKey(),
entry.getValue()

View File

@@ -216,49 +216,46 @@ public class Subflow extends Task implements ExecutableTask<Subflow.Output>, Chi
VariablesService variablesService = ((DefaultRunContext) runContext).getApplicationContext().getBean(VariablesService.class);
if (this.wait) { // we only compute outputs if we wait for the subflow
boolean isOutputsAllowed = runContext
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
.orElse(true);
List<io.kestra.core.models.flows.Output> subflowOutputs = flow.getOutputs();
// region [deprecated] Subflow outputs feature
if (subflowOutputs == null && isOutputsAllowed && this.getOutputs() != null) {
subflowOutputs = this.getOutputs().entrySet().stream()
.<io.kestra.core.models.flows.Output>map(entry -> io.kestra.core.models.flows.Output
.builder()
.id(entry.getKey())
.value(entry.getValue())
.required(true)
.build()
)
.toList();
if (subflowOutputs == null && this.getOutputs() != null) {
boolean isOutputsAllowed = runContext
.<Boolean>pluginConfiguration(PLUGIN_FLOW_OUTPUTS_ENABLED)
.orElse(true);
if (isOutputsAllowed) {
try {
subflowOutputs = this.getOutputs().entrySet().stream()
.<io.kestra.core.models.flows.Output>map(entry -> io.kestra.core.models.flows.Output
.builder()
.id(entry.getKey())
.value(entry.getValue())
.required(true)
.build()
)
.toList();
} catch (Exception e) {
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
return failSubflowDueToOutput(runContext, taskRun, execution, e, variables);
}
} else {
runContext.logger().warn("Defining outputs inside the Subflow task is not allowed.");
}
}
//endregion
if (subflowOutputs != null && !subflowOutputs.isEmpty()) {
try {
Map<String, Object> outputs = FlowInputOutput.renderFlowOutputs(subflowOutputs, runContext);
Map<String, Object> rOutputs = FlowInputOutput.renderFlowOutputs(subflowOutputs, runContext);
FlowInputOutput flowInputOutput = ((DefaultRunContext)runContext).getApplicationContext().getBean(FlowInputOutput.class); // this is hacking
if (flow.getOutputs() != null && flowInputOutput != null) {
outputs = flowInputOutput.typedOutputs(flow, execution, outputs);
rOutputs = flowInputOutput.typedOutputs(flow, execution, rOutputs);
}
builder.outputs(outputs);
builder.outputs(rOutputs);
} catch (Exception e) {
runContext.logger().warn("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
var state = State.Type.fail(this);
Variables variables = variablesService.of(StorageContext.forTask(taskRun), builder.build());
taskRun = taskRun
.withState(state)
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
.withOutputs(variables);
return Optional.of(SubflowExecutionResult.builder()
.executionId(execution.getId())
.state(State.Type.FAILED)
.parentTaskRun(taskRun)
.build());
return failSubflowDueToOutput(runContext, taskRun, execution, e, variables);
}
}
}
@@ -282,6 +279,21 @@ public class Subflow extends Task implements ExecutableTask<Subflow.Output>, Chi
return Optional.of(ExecutableUtils.subflowExecutionResult(taskRun, execution));
}
private Optional<SubflowExecutionResult> failSubflowDueToOutput(RunContext runContext, TaskRun taskRun, Execution execution, Exception e, Variables outputs) {
runContext.logger().error("Failed to extract outputs with the error: '{}'", e.getLocalizedMessage(), e);
var state = State.Type.fail(this);
taskRun = taskRun
.withState(state)
.withAttempts(Collections.singletonList(TaskRunAttempt.builder().state(new State().withState(state)).build()))
.withOutputs(outputs);
return Optional.of(SubflowExecutionResult.builder()
.executionId(execution.getId())
.state(State.Type.FAILED)
.parentTaskRun(taskRun)
.build());
}
@Override
public boolean waitForExecution() {
return this.wait;

View File

@@ -202,7 +202,7 @@ import static io.kestra.core.utils.Rethrow.throwPredicate;
code = """
id: sentry_execution_example
namespace: company.team
tasks:
- id: send_alert
type: io.kestra.plugin.notifications.sentry.SentryExecution
@@ -221,7 +221,7 @@ import static io.kestra.core.utils.Rethrow.throwPredicate;
- WARNING
- type: io.kestra.plugin.core.condition.ExecutionNamespace
namespace: company.payroll
prefix: false"""
prefix: false"""
)
},
@@ -405,6 +405,28 @@ public class Flow extends AbstractTrigger implements TriggerOutput<Flow.Output>
return conditions;
}
@JsonIgnore
public Map<String, Condition> getUpstreamFlowsConditions() {
AtomicInteger conditionId = new AtomicInteger();
return ListUtils.emptyOnNull(flows).stream()
.map(upstreamFlow -> Map.entry(
"condition_" + conditionId.incrementAndGet(),
new UpstreamFlowCondition(upstreamFlow)
))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
@JsonIgnore
public Map<String, Condition> getWhereConditions() {
AtomicInteger conditionId = new AtomicInteger();
return ListUtils.emptyOnNull(where).stream()
.map(filter -> Map.entry(
"condition_" + conditionId.incrementAndGet() + "_" + filter.getId(),
new FilterCondition(filter)
))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
@Override
public Logger logger() {
return log;

View File

@@ -21,10 +21,13 @@ import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
@Execution(ExecutionMode.SAME_THREAD)
class DocumentationGeneratorTest {
@Inject
JsonSchemaGenerator jsonSchemaGenerator;

View File

@@ -0,0 +1,121 @@
package io.kestra.core.events;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
class CrudEventTest {
@Test
void shouldReturnCreateEventWhenModelIsProvided() {
// Given
String model = "testModel";
// When
CrudEvent<String> event = CrudEvent.create(model);
// Then
assertThat(event.getModel()).isEqualTo(model);
assertThat(event.getPreviousModel()).isNull();
assertThat(event.getType()).isEqualTo(CrudEventType.CREATE);
assertThat(event.getRequest()).isNull();
}
@Test
void shouldThrowExceptionWhenCreateEventWithNullModel() {
// Given
String model = null;
// When / Then
assertThatThrownBy(() -> CrudEvent.create(model))
.isInstanceOf(NullPointerException.class)
.hasMessage("Can't create CREATE event with a null model");
}
@Test
void shouldReturnDeleteEventWhenModelIsProvided() {
// Given
String model = "testModel";
// When
CrudEvent<String> event = CrudEvent.delete(model);
// Then
assertThat(event.getModel()).isNull();
assertThat(event.getPreviousModel()).isEqualTo(model);
assertThat(event.getType()).isEqualTo(CrudEventType.DELETE);
assertThat(event.getRequest()).isNull();
}
@Test
void shouldThrowExceptionWhenDeleteEventWithNullModel() {
// Given
String model = null;
// When / Then
assertThatThrownBy(() -> CrudEvent.delete(model))
.isInstanceOf(NullPointerException.class)
.hasMessage("Can't create DELETE event with a null model");
}
@Test
void shouldReturnUpdateEventWhenBeforeAndAfterAreProvided() {
// Given
String before = "oldModel";
String after = "newModel";
// When
CrudEvent<String> event = CrudEvent.of(before, after);
// Then
assertThat(event.getModel()).isEqualTo(after);
assertThat(event.getPreviousModel()).isEqualTo(before);
assertThat(event.getType()).isEqualTo(CrudEventType.UPDATE);
assertThat(event.getRequest()).isNull();
}
@Test
void shouldReturnCreateEventWhenBeforeIsNullAndAfterIsProvided() {
// Given
String before = null;
String after = "newModel";
// When
CrudEvent<String> event = CrudEvent.of(before, after);
// Then
assertThat(event.getModel()).isEqualTo(after);
assertThat(event.getPreviousModel()).isNull();
assertThat(event.getType()).isEqualTo(CrudEventType.CREATE);
assertThat(event.getRequest()).isNull();
}
@Test
void shouldReturnDeleteEventWhenAfterIsNullAndBeforeIsProvided() {
// Given
String before = "oldModel";
String after = null;
// When
CrudEvent<String> event = CrudEvent.of(before, after);
// Then
assertThat(event.getModel()).isNull();
assertThat(event.getPreviousModel()).isEqualTo(before);
assertThat(event.getType()).isEqualTo(CrudEventType.DELETE);
assertThat(event.getRequest()).isNull();
}
@Test
void shouldThrowExceptionWhenBothBeforeAndAfterAreNull() {
// Given
String before = null;
String after = null;
// When / Then
assertThatThrownBy(() -> CrudEvent.of(before, after))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Both before and after cannot be null");
}
}

View File

@@ -37,6 +37,7 @@ import lombok.Value;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -67,6 +68,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
@KestraTest
@Testcontainers
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
class HttpClientTest {
@Inject
private ApplicationContext applicationContext;

View File

@@ -1,19 +1,32 @@
package io.kestra.core.models;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.validations.ModelValidator;
import jakarta.inject.Inject;
import jakarta.validation.ConstraintViolationException;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
class LabelTest {
@Inject
private ModelValidator modelValidator;
@Test
void shouldGetNestedMapGivenDistinctLabels() {
Map<String, Object> result = Label.toNestedMap(List.of(
new Label(Label.USERNAME, "test"),
new Label(Label.CORRELATION_ID, "id"))
new Label(Label.CORRELATION_ID, "id"),
new Label("", "bar"),
new Label(null, "bar"),
new Label("foo", ""),
new Label("baz", null)
)
);
assertThat(result).isEqualTo(
@@ -34,6 +47,18 @@ class LabelTest {
);
}
@Test
void toNestedMapShouldIgnoreEmptyOrNull() {
Map<String, Object> result = Label.toNestedMap(List.of(
new Label("", "bar"),
new Label(null, "bar"),
new Label("foo", ""),
new Label("baz", null))
);
assertThat(result).isEmpty();
}
@Test
void shouldGetMapGivenDistinctLabels() {
Map<String, String> result = Label.toMap(List.of(
@@ -59,6 +84,18 @@ class LabelTest {
);
}
@Test
void toMapShouldIgnoreEmptyOrNull() {
Map<String, String> result = Label.toMap(List.of(
new Label("", "bar"),
new Label(null, "bar"),
new Label("foo", ""),
new Label("baz", null))
);
assertThat(result).isEmpty();
}
@Test
void shouldDuplicateLabelsWithKeyOrderKept() {
List<Label> result = Label.deduplicate(List.of(
@@ -73,4 +110,28 @@ class LabelTest {
new Label(Label.CORRELATION_ID, "id")
);
}
@Test
void deduplicateShouldIgnoreEmptyAndNull() {
List<Label> result = Label.deduplicate(List.of(
new Label("", "bar"),
new Label(null, "bar"),
new Label("foo", ""),
new Label("baz", null))
);
assertThat(result).isEmpty();
}
@Test
void shouldValidateEmpty() {
Optional<ConstraintViolationException> validLabelResult = modelValidator.isValid(new Label("foo", "bar"));
assertThat(validLabelResult.isPresent()).isFalse();
Optional<ConstraintViolationException> emptyValueLabelResult = modelValidator.isValid(new Label("foo", ""));
assertThat(emptyValueLabelResult.isPresent()).isTrue();
Optional<ConstraintViolationException> emptyKeyLabelResult = modelValidator.isValid(new Label("", "bar"));
assertThat(emptyKeyLabelResult.isPresent()).isTrue();
}
}

View File

@@ -13,19 +13,19 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
class ExecutionTest {
private static final TaskRun.TaskRunBuilder TASK_RUN = TaskRun.builder()
.id("test");
@Test
void hasTaskRunJoinableTrue() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State(State.Type.RUNNING, new State()))
.build())
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State(State.Type.FAILED, new State()
.withState(State.Type.RUNNING)
))
@@ -36,13 +36,15 @@ class ExecutionTest {
@Test
void hasTaskRunJoinableSameState() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State())
.build())
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State())
.build()
)).isFalse();
@@ -51,7 +53,8 @@ class ExecutionTest {
@Test
void hasTaskRunJoinableFailedExecutionFromExecutor() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State(State.Type.FAILED, new State()
.withState(State.Type.RUNNING)
))
@@ -59,7 +62,8 @@ class ExecutionTest {
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State(State.Type.RUNNING, new State()))
.build()
)).isFalse();
@@ -68,7 +72,8 @@ class ExecutionTest {
@Test
void hasTaskRunJoinableRestartFailed() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State(State.Type.CREATED, new State()
.withState(State.Type.RUNNING)
.withState(State.Type.FAILED)
@@ -77,7 +82,8 @@ class ExecutionTest {
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State(State.Type.FAILED, new State()
.withState(State.Type.RUNNING)
))
@@ -88,7 +94,8 @@ class ExecutionTest {
@Test
void hasTaskRunJoinableRestartSuccess() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State(State.Type.CREATED, new State()
.withState(State.Type.RUNNING)
.withState(State.Type.SUCCESS)
@@ -97,7 +104,8 @@ class ExecutionTest {
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State(State.Type.SUCCESS, new State()
.withState(State.Type.RUNNING)
.withState(State.Type.SUCCESS)
@@ -109,7 +117,8 @@ class ExecutionTest {
@Test
void hasTaskRunJoinableAfterRestart() {
Execution execution = Execution.builder()
.taskRunList(Collections.singletonList(TASK_RUN
.taskRunList(Collections.singletonList(TaskRun.builder()
.id("test")
.state(new State(State.Type.CREATED, new State()
.withState(State.Type.RUNNING)
.withState(State.Type.FAILED)
@@ -118,7 +127,8 @@ class ExecutionTest {
)
.build();
assertThat(execution.hasTaskRunJoinable(TASK_RUN
assertThat(execution.hasTaskRunJoinable(TaskRun.builder()
.id("test")
.state(new State(State.Type.SUCCESS, new State()
.withState(State.Type.RUNNING)
.withState(State.Type.FAILED)

View File

@@ -7,12 +7,11 @@ import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.triggers.Trigger;
import io.kestra.core.queues.QueueException;
import io.kestra.core.repositories.TriggerRepositoryInterface;
import io.kestra.core.runners.RunnerUtils;
import io.kestra.core.runners.TestRunnerUtils;
import io.kestra.core.serializers.YamlParser;
import io.kestra.core.services.GraphService;
import io.kestra.core.utils.GraphUtils;
@@ -45,7 +44,7 @@ class FlowGraphTest {
private TriggerRepositoryInterface triggerRepositoryInterface;
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
@Test
void simple() throws IllegalVariableEvaluationException, IOException {
@@ -261,10 +260,10 @@ class FlowGraphTest {
}
@Test
@LoadFlows({"flows/valids/task-flow.yaml",
"flows/valids/switch.yaml"})
@LoadFlows(value = {"flows/valids/task-flow.yaml",
"flows/valids/switch.yaml"}, tenantId = "tenant1")
void subflow() throws IllegalVariableEvaluationException, IOException, FlowProcessingException {
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml");
FlowWithSource flow = this.parse("flows/valids/task-flow.yaml", "tenant1");
FlowGraph flowGraph = GraphUtils.flowGraph(flow, null);
assertThat(flowGraph.getNodes().size()).isEqualTo(6);
@@ -293,15 +292,15 @@ class FlowGraphTest {
}
@Test
@LoadFlows({"flows/valids/task-flow-dynamic.yaml",
"flows/valids/switch.yaml"})
@LoadFlows(value = {"flows/valids/task-flow-dynamic.yaml",
"flows/valids/switch.yaml"}, tenantId = "tenant2")
void dynamicIdSubflow() throws IllegalVariableEvaluationException, TimeoutException, QueueException, IOException, FlowProcessingException {
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml").toBuilder().revision(1).build();
FlowWithSource flow = this.parse("flows/valids/task-flow-dynamic.yaml", "tenant2").toBuilder().revision(1).build();
IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> graphService.flowGraph(flow, Collections.singletonList("root.launch")));
assertThat(illegalArgumentException.getMessage()).isEqualTo("Can't expand subflow task 'launch' because namespace and/or flowId contains dynamic values. This can only be viewed on an execution.");
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
Execution execution = runnerUtils.runOne("tenant2", "io.kestra.tests", "task-flow-dynamic", 1, (f, e) -> Map.of(
"namespace", f.getNamespace(),
"flowId", "switch"
));
@@ -373,13 +372,17 @@ class FlowGraphTest {
}
private FlowWithSource parse(String path) throws IOException {
return parse(path, MAIN_TENANT);
}
private FlowWithSource parse(String path, String tenantId) throws IOException {
URL resource = TestsUtils.class.getClassLoader().getResource(path);
assert resource != null;
File file = new File(resource.getFile());
return YamlParser.parse(file, FlowWithSource.class).toBuilder()
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.source(Files.readString(file.toPath()))
.build();
}

View File

@@ -4,6 +4,7 @@ import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.runners.*;
import io.kestra.core.storages.StorageContext;
import io.kestra.core.storages.StorageInterface;
import io.kestra.core.utils.IdUtils;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
@@ -84,8 +85,9 @@ class URIFetcherTest {
@Test
void shouldFetchFromNsfile() throws IOException {
URI uri = createNsFile(false);
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "namespace")));
String namespace = IdUtils.create();
URI uri = createNsFile(namespace, false);
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", namespace)));
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
String fetchedContent = new String(fetch.readAllBytes());
@@ -95,7 +97,8 @@ class URIFetcherTest {
@Test
void shouldFetchFromNsfileFromOtherNs() throws IOException {
URI uri = createNsFile(true);
String namespace = IdUtils.create();
URI uri = createNsFile(namespace, true);
RunContext runContext = runContextFactory.of(Map.of("flow", Map.of("namespace", "other")));
try (var fetch = URIFetcher.of(uri).fetch(runContext)) {
@@ -139,8 +142,7 @@ class URIFetcherTest {
);
}
private URI createNsFile(boolean nsInAuthority) throws IOException {
String namespace = "namespace";
private URI createNsFile(String namespace, boolean nsInAuthority) throws IOException {
String filePath = "file.txt";
storage.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
storage.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));

View File

@@ -10,6 +10,7 @@ import io.kestra.core.models.tasks.Task;
import io.kestra.core.runners.RunContext;
import io.kestra.core.runners.RunContextFactory;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.utils.IdUtils;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Test;
@@ -33,17 +34,15 @@ class ScriptServiceTest {
@Test
void replaceInternalStorage() throws IOException {
var runContext = runContextFactory.of();
String tenant = IdUtils.create();
var runContext = runContextFactory.of("id", "namespace", tenant);
var command = ScriptService.replaceInternalStorage(runContext, null, false);
assertThat(command).isEqualTo("");
command = ScriptService.replaceInternalStorage(runContext, "my command", false);
assertThat(command).isEqualTo("my command");
Path path = Path.of("/tmp/unittest/main/file.txt");
if (!path.toFile().exists()) {
Files.createFile(path);
}
Path path = createFile(tenant, "file");
String internalStorageUri = "kestra://some/file.txt";
File localFile = null;
@@ -70,12 +69,10 @@ class ScriptServiceTest {
@Test
void replaceInternalStorageUnicode() throws IOException {
var runContext = runContextFactory.of();
String tenant = IdUtils.create();
var runContext = runContextFactory.of("id", "namespace", tenant);
Path path = Path.of("/tmp/unittest/main/file-龍.txt");
if (!path.toFile().exists()) {
Files.createFile(path);
}
Path path = createFile(tenant, "file-龍");
String internalStorageUri = "kestra://some/file-龍.txt";
File localFile = null;
@@ -95,12 +92,10 @@ class ScriptServiceTest {
@Test
void uploadInputFiles() throws IOException {
var runContext = runContextFactory.of();
String tenant = IdUtils.create();
var runContext = runContextFactory.of("id", "namespace", tenant);
Path path = Path.of("/tmp/unittest/main/file.txt");
if (!path.toFile().exists()) {
Files.createFile(path);
}
Path path = createFile(tenant, "file");
List<File> filesToDelete = new ArrayList<>();
String internalStorageUri = "kestra://some/file.txt";
@@ -143,13 +138,11 @@ class ScriptServiceTest {
@Test
void uploadOutputFiles() throws IOException {
var runContext = runContextFactory.of();
Path path = Path.of("/tmp/unittest/main/file.txt");
if (!path.toFile().exists()) {
Files.createFile(path);
}
String tenant = IdUtils.create();
var runContext = runContextFactory.of("id", "namespace", tenant);
Path path = createFile(tenant, "file");
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/main"));
var outputFiles = ScriptService.uploadOutputFiles(runContext, Path.of("/tmp/unittest/%s".formatted(tenant)));
assertThat(outputFiles, not(anEmptyMap()));
assertThat(outputFiles.get("file.txt")).isEqualTo(URI.create("kestra:///file.txt"));
@@ -232,4 +225,13 @@ class ScriptServiceTest {
.build();
return runContextFactory.of(flow, task, execution, taskRun);
}
private static Path createFile(String tenant, String fileName) throws IOException {
Path path = Path.of("/tmp/unittest/%s/%s.txt".formatted(tenant, fileName));
if (!path.toFile().exists()) {
Files.createDirectory(Path.of("/tmp/unittest/%s".formatted(tenant)));
Files.createFile(path);
}
return path;
}
}

View File

@@ -3,6 +3,7 @@ package io.kestra.core.models.triggers.multipleflows;
import com.google.common.collect.ImmutableMap;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.property.Property;
import io.kestra.core.utils.TestsUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Test;
import io.kestra.plugin.core.condition.ExecutionFlow;
@@ -33,8 +34,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void allDefault() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
@@ -50,8 +52,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void daily() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofSeconds(0)).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
@@ -67,8 +70,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void dailyAdvance() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofDays(1)).windowAdvance(Duration.ofHours(4).negated()).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
@@ -84,8 +88,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void hourly() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofHours(1)).windowAdvance(Duration.ofHours(4).negated()).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
@@ -102,8 +107,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void minutely() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofMinutes(15)).windowAdvance(Duration.ofMinutes(5).negated()).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
@@ -115,8 +121,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void expiration() throws Exception {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -136,8 +143,9 @@ public abstract class AbstractMultipleConditionStorageTest {
@Test
void expired() throws Exception {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().window(Duration.ofSeconds(2)).windowAdvance(Duration.ofMinutes(0).negated()).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -146,20 +154,21 @@ public abstract class AbstractMultipleConditionStorageTest {
assertThat(window.getResults().get("a")).isTrue();
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isZero();
Thread.sleep(2005);
expired = multipleConditionStorage.expired(null);
expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isEqualTo(1);
}
@Test
void dailyTimeDeadline() throws Exception {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().plusSeconds(2)).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -168,20 +177,21 @@ public abstract class AbstractMultipleConditionStorageTest {
assertThat(window.getResults().get("a")).isTrue();
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isZero();
Thread.sleep(2005);
expired = multipleConditionStorage.expired(null);
expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isEqualTo(1);
}
@Test
void dailyTimeDeadline_Expired() throws Exception {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_DEADLINE).deadline(LocalTime.now().minusSeconds(1)).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -190,16 +200,17 @@ public abstract class AbstractMultipleConditionStorageTest {
assertThat(window.getResults()).isEmpty();
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isEqualTo(1);
}
@Test
void dailyTimeWindow() throws Exception {
void dailyTimeWindow() {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
LocalTime startTime = LocalTime.now().truncatedTo(ChronoUnit.MINUTES);
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.DAILY_TIME_WINDOW).startTime(startTime).endTime(startTime.plusMinutes(5)).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -208,15 +219,16 @@ public abstract class AbstractMultipleConditionStorageTest {
assertThat(window.getResults().get("a")).isTrue();
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isZero();
}
@Test
void slidingWindow() throws Exception {
MultipleConditionStorageInterface multipleConditionStorage = multipleConditionStorage();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Pair<Flow, MultipleCondition> pair = mockFlow(TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
Pair<Flow, MultipleCondition> pair = mockFlow(tenant, TimeWindow.builder().type(Type.SLIDING_WINDOW).window(Duration.ofHours(1)).build());
MultipleConditionWindow window = multipleConditionStorage.getOrCreate(pair.getKey(), pair.getRight(), Collections.emptyMap());
this.save(multipleConditionStorage, pair.getLeft(), Collections.singletonList(window.with(ImmutableMap.of("a", true))));
@@ -225,13 +237,13 @@ public abstract class AbstractMultipleConditionStorageTest {
assertThat(window.getResults().get("a")).isTrue();
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(null);
List<MultipleConditionWindow> expired = multipleConditionStorage.expired(tenant);
assertThat(expired.size()).isZero();
}
private static Pair<Flow, MultipleCondition> mockFlow(TimeWindow sla) {
private static Pair<Flow, MultipleCondition> mockFlow(String tenantId, TimeWindow sla) {
var multipleCondition = MultipleCondition.builder()
.id("condition-multiple")
.id("condition-multiple-%s".formatted(tenantId))
.conditions(ImmutableMap.of(
"flow-a", ExecutionFlow.builder()
.flowId(Property.ofValue("flow-a"))
@@ -248,6 +260,7 @@ public abstract class AbstractMultipleConditionStorageTest {
Flow flow = Flow.builder()
.namespace(NAMESPACE)
.id("multiple-flow")
.tenantId(tenantId)
.revision(1)
.triggers(Collections.singletonList(io.kestra.plugin.core.trigger.Flow.builder()
.id("trigger-flow")

View File

@@ -13,21 +13,20 @@ import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
public abstract class AbstractFeatureUsageReportTest {
@Inject
FeatureUsageReport featureUsageReport;
@Test
public void shouldGetReport() {
// When
Instant now = Instant.now();
FeatureUsageReport.UsageEvent event = featureUsageReport.report(
now,
now,
Reportable.TimeInterval.of(now.minus(Duration.ofDays(1)).atZone(ZoneId.systemDefault()), now.atZone(ZoneId.systemDefault()))
);
// Then
assertThat(event.getExecutions().getDailyExecutionsCount().size()).isGreaterThan(0);
assertThat(event.getExecutions().getDailyTaskRunsCount()).isNull();
}
}

View File

@@ -18,10 +18,10 @@ import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
class SystemInformationReportTest {
@Inject
private SystemInformationReport systemInformationReport;
@Test
void shouldGetReport() {
SystemInformationReport.SystemInformationEvent event = systemInformationReport.report(Instant.now());
@@ -32,34 +32,34 @@ class SystemInformationReportTest {
assertThat(event.host().getHardware().getLogicalProcessorCount()).isNotNull();
assertThat(event.host().getJvm().getName()).isNotNull();
assertThat(event.host().getOs().getFamily()).isNotNull();
assertThat(event.configurations().getRepositoryType()).isEqualTo("memory");
assertThat(event.configurations().getQueueType()).isEqualTo("memory");
assertThat(event.configurations().getRepositoryType()).isEqualTo("h2");
assertThat(event.configurations().getQueueType()).isEqualTo("h2");
}
@MockBean(SettingRepositoryInterface.class)
@Singleton
static class TestSettingRepository implements SettingRepositoryInterface {
public static Object UUID = null;
@Override
public Optional<Setting> findByKey(String key) {
return Optional.empty();
}
@Override
public List<Setting> findAll() {
return new ArrayList<>();
}
@Override
public Setting save(Setting setting) throws ConstraintViolationException {
if (setting.getKey().equals(Setting.INSTANCE_UUID)) {
UUID = setting.getValue();
}
return setting;
}
@Override
public Setting delete(Setting setting) {
return setting;

View File

@@ -25,6 +25,7 @@ import io.kestra.core.models.tasks.ResolvedTask;
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.NamespaceUtils;
import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.core.dashboard.data.Executions;
import io.kestra.plugin.core.debug.Return;
import io.micronaut.data.model.Pageable;
@@ -48,7 +49,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static io.kestra.core.models.flows.FlowScope.USER;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doReturn;
@@ -62,17 +62,17 @@ public abstract class AbstractExecutionRepositoryTest {
@Inject
protected ExecutionRepositoryInterface executionRepository;
public static Execution.ExecutionBuilder builder(State.Type state, String flowId) {
return builder(state, flowId, NAMESPACE);
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId) {
return builder(tenantId, state, flowId, NAMESPACE);
}
public static Execution.ExecutionBuilder builder(State.Type state, String flowId, String namespace) {
public static Execution.ExecutionBuilder builder(String tenantId, State.Type state, String flowId, String namespace) {
State finalState = randomDuration(state);
Execution.ExecutionBuilder execution = Execution.builder()
.id(FriendlyId.createFriendlyId())
.namespace(namespace)
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.flowId(flowId == null ? FLOW : flowId)
.flowRevision(1)
.state(finalState);
@@ -126,11 +126,11 @@ public abstract class AbstractExecutionRepositoryTest {
return finalState;
}
protected void inject() {
inject(null);
protected void inject(String tenantId) {
inject(tenantId, null);
}
protected void inject(String executionTriggerId) {
protected void inject(String tenantId, String executionTriggerId) {
ExecutionTrigger executionTrigger = null;
if (executionTriggerId != null) {
@@ -139,7 +139,7 @@ public abstract class AbstractExecutionRepositoryTest {
.build();
}
executionRepository.save(builder(State.Type.RUNNING, null)
executionRepository.save(builder(tenantId, State.Type.RUNNING, null)
.labels(List.of(
new Label("key", "value"),
new Label("key2", "value2")
@@ -149,6 +149,7 @@ public abstract class AbstractExecutionRepositoryTest {
);
for (int i = 1; i < 28; i++) {
executionRepository.save(builder(
tenantId,
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
i < 15 ? null : "second"
).trigger(executionTrigger).build());
@@ -156,6 +157,7 @@ public abstract class AbstractExecutionRepositoryTest {
// add a test execution, this should be ignored in search & statistics
executionRepository.save(builder(
tenantId,
State.Type.SUCCESS,
null
)
@@ -167,9 +169,10 @@ public abstract class AbstractExecutionRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all(QueryFilter filter, int expectedSize){
inject("executionTriggerId");
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant, "executionTriggerId");
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
ArrayListTotal<Execution> entries = executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
assertThat(entries).hasSize(expectedSize);
}
@@ -192,7 +195,8 @@ public abstract class AbstractExecutionRepositoryTest {
@ParameterizedTest
@MethodSource("errorFilterCombinations")
void should_fail_to_find_all(QueryFilter filter){
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
assertThrows(InvalidQueryFiltersException.class, () -> executionRepository.find(Pageable.UNPAGED, tenant, List.of(filter)));
}
static Stream<QueryFilter> errorFilterCombinations() {
@@ -208,9 +212,10 @@ public abstract class AbstractExecutionRepositoryTest {
@Test
protected void find() {
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant);
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
assertThat(executions.getTotal()).isEqualTo(28L);
assertThat(executions.size()).isEqualTo(10);
@@ -219,7 +224,7 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value( List.of(State.Type.RUNNING, State.Type.FAILED))
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(8L);
filters = List.of(QueryFilter.builder()
@@ -227,7 +232,7 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value(Map.of("key", "value"))
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(1L);
filters = List.of(QueryFilter.builder()
@@ -235,7 +240,7 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value(Map.of("key", "value2"))
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(0L);
filters = List.of(QueryFilter.builder()
@@ -244,7 +249,7 @@ public abstract class AbstractExecutionRepositoryTest {
.value(Map.of("key", "value", "keyTest", "valueTest"))
.build()
);
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(0L);
filters = List.of(QueryFilter.builder()
@@ -252,7 +257,7 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value("second")
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(13L);
filters = List.of(QueryFilter.builder()
@@ -266,7 +271,7 @@ public abstract class AbstractExecutionRepositoryTest {
.value(NAMESPACE)
.build()
);
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(13L);
filters = List.of(QueryFilter.builder()
@@ -274,7 +279,7 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.STARTS_WITH)
.value("io.kestra")
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(28L);
}
@@ -282,15 +287,16 @@ public abstract class AbstractExecutionRepositoryTest {
protected void findTriggerExecutionId() {
String executionTriggerId = IdUtils.create();
inject(executionTriggerId);
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant, executionTriggerId);
inject(tenant);
var filters = List.of(QueryFilter.builder()
.field(QueryFilter.Field.TRIGGER_EXECUTION_ID)
.operation(QueryFilter.Op.EQUALS)
.value(executionTriggerId)
.build());
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(28L);
assertThat(executions.size()).isEqualTo(10);
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
@@ -300,7 +306,7 @@ public abstract class AbstractExecutionRepositoryTest {
.value(ExecutionRepositoryInterface.ChildFilter.CHILD)
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(28L);
assertThat(executions.size()).isEqualTo(10);
assertThat(executions.getFirst().getTrigger().getVariables().get("executionId")).isEqualTo(executionTriggerId);
@@ -311,20 +317,21 @@ public abstract class AbstractExecutionRepositoryTest {
.value(ExecutionRepositoryInterface.ChildFilter.MAIN)
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters );
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters );
assertThat(executions.getTotal()).isEqualTo(28L);
assertThat(executions.size()).isEqualTo(10);
assertThat(executions.getFirst().getTrigger()).isNull();
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, null);
executions = executionRepository.find(Pageable.from(1, 10), tenant, null);
assertThat(executions.getTotal()).isEqualTo(56L);
}
@Test
protected void findWithSort() {
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant);
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), MAIN_TENANT, null);
ArrayListTotal<Execution> executions = executionRepository.find(Pageable.from(1, 10, Sort.of(Sort.Order.desc("id"))), tenant, null);
assertThat(executions.getTotal()).isEqualTo(28L);
assertThat(executions.size()).isEqualTo(10);
@@ -333,100 +340,92 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value(List.of(State.Type.RUNNING, State.Type.FAILED))
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.getTotal()).isEqualTo(8L);
}
@Test
protected void findTaskRun() {
inject();
ArrayListTotal<TaskRun> taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, null);
assertThat(taskRuns.getTotal()).isEqualTo(74L);
assertThat(taskRuns.size()).isEqualTo(10);
var filters = List.of(QueryFilter.builder()
.field(QueryFilter.Field.LABELS)
.operation(QueryFilter.Op.EQUALS)
.value(Map.of("key", "value"))
.build());
taskRuns = executionRepository.findTaskRun(Pageable.from(1, 10), MAIN_TENANT, filters);
assertThat(taskRuns.getTotal()).isEqualTo(1L);
assertThat(taskRuns.size()).isEqualTo(1);
}
@Test
protected void findById() {
executionRepository.save(ExecutionFixture.EXECUTION_1);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
executionRepository.save(execution1);
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
assertThat(full.isPresent()).isTrue();
full.ifPresent(current -> {
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_1.getId());
assertThat(full.get().getId()).isEqualTo(execution1.getId());
});
}
@Test
protected void shouldFindByIdTestExecution() {
executionRepository.save(ExecutionFixture.EXECUTION_TEST);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
var executionTest = ExecutionFixture.EXECUTION_TEST(tenant);
executionRepository.save(executionTest);
Optional<Execution> full = executionRepository.findById(null, ExecutionFixture.EXECUTION_TEST.getId());
Optional<Execution> full = executionRepository.findById(tenant, executionTest.getId());
assertThat(full.isPresent()).isTrue();
full.ifPresent(current -> {
assertThat(full.get().getId()).isEqualTo(ExecutionFixture.EXECUTION_TEST.getId());
assertThat(full.get().getId()).isEqualTo(executionTest.getId());
});
}
@Test
protected void purge() {
executionRepository.save(ExecutionFixture.EXECUTION_1);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
executionRepository.save(execution1);
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
assertThat(full.isPresent()).isTrue();
executionRepository.purge(ExecutionFixture.EXECUTION_1);
executionRepository.purge(execution1);
full = executionRepository.findById(null, ExecutionFixture.EXECUTION_1.getId());
full = executionRepository.findById(tenant, execution1.getId());
assertThat(full.isPresent()).isFalse();
}
@Test
protected void delete() {
executionRepository.save(ExecutionFixture.EXECUTION_1);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
var execution1 = ExecutionFixture.EXECUTION_1(tenant);
executionRepository.save(execution1);
Optional<Execution> full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
Optional<Execution> full = executionRepository.findById(tenant, execution1.getId());
assertThat(full.isPresent()).isTrue();
executionRepository.delete(ExecutionFixture.EXECUTION_1);
executionRepository.delete(execution1);
full = executionRepository.findById(MAIN_TENANT, ExecutionFixture.EXECUTION_1.getId());
full = executionRepository.findById(tenant, execution1.getId());
assertThat(full.isPresent()).isFalse();
}
@Test
protected void mappingConflict() {
executionRepository.save(ExecutionFixture.EXECUTION_2);
executionRepository.save(ExecutionFixture.EXECUTION_1);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
executionRepository.save(ExecutionFixture.EXECUTION_2(tenant));
executionRepository.save(ExecutionFixture.EXECUTION_1(tenant));
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(MAIN_TENANT, NAMESPACE, FLOW, Pageable.from(1, 10));
ArrayListTotal<Execution> page1 = executionRepository.findByFlowId(tenant, NAMESPACE, FLOW, Pageable.from(1, 10));
assertThat(page1.size()).isEqualTo(2);
}
@Test
protected void dailyStatistics() throws InterruptedException {
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
for (int i = 0; i < 28; i++) {
executionRepository.save(builder(
tenant,
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
i < 15 ? null : "second"
).build());
}
executionRepository.save(builder(
tenant,
State.Type.SUCCESS,
"second"
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
@@ -436,15 +435,14 @@ public abstract class AbstractExecutionRepositoryTest {
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
tenant,
null,
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
false);
null);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
@@ -456,131 +454,52 @@ public abstract class AbstractExecutionRepositoryTest {
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
tenant,
List.of(FlowScope.USER, FlowScope.SYSTEM),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
false);
null);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(21L);
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
tenant,
List.of(FlowScope.USER),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
false);
null);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(20L);
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
tenant,
List.of(FlowScope.SYSTEM),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
false);
null);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(1L);
}
@Test
protected void taskRunsDailyStatistics() {
for (int i = 0; i < 28; i++) {
executionRepository.save(builder(
i < 5 ? State.Type.RUNNING : (i < 8 ? State.Type.FAILED : State.Type.SUCCESS),
i < 15 ? null : "second"
).build());
}
executionRepository.save(builder(
State.Type.SUCCESS,
"second"
).namespace(NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE).build());
List<DailyExecutionStatistics> result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
null,
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
true);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().size()).isEqualTo(11);
assertThat(result.get(10).getDuration().getAvg().toMillis()).isGreaterThan(0L);
assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED)).isEqualTo(3L * 2);
assertThat(result.get(10).getExecutionCounts().get(State.Type.RUNNING)).isEqualTo(5L * 2);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
List.of(FlowScope.USER, FlowScope.SYSTEM),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
true);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(57L);
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
List.of(FlowScope.USER),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
true);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(55L);
result = executionRepository.dailyStatistics(
null,
MAIN_TENANT,
List.of(FlowScope.SYSTEM),
null,
null,
ZonedDateTime.now().minusDays(10),
ZonedDateTime.now(),
null,
null,
true);
assertThat(result.size()).isEqualTo(11);
assertThat(result.get(10).getExecutionCounts().get(State.Type.SUCCESS)).isEqualTo(2L);
}
@SuppressWarnings("OptionalGetWithoutIsPresent")
@Test
protected void executionsCount() throws InterruptedException {
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
for (int i = 0; i < 14; i++) {
executionRepository.save(builder(
tenant,
State.Type.SUCCESS,
i < 2 ? "first" : (i < 5 ? "second" : "third")
).build());
@@ -590,7 +509,7 @@ public abstract class AbstractExecutionRepositoryTest {
Thread.sleep(500);
List<ExecutionCount> result = executionRepository.executionCounts(
MAIN_TENANT,
tenant,
List.of(
new Flow(NAMESPACE, "first"),
new Flow(NAMESPACE, "second"),
@@ -609,7 +528,7 @@ public abstract class AbstractExecutionRepositoryTest {
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("missing")).findFirst().get().getCount()).isEqualTo(0L);
result = executionRepository.executionCounts(
MAIN_TENANT,
tenant,
List.of(
new Flow(NAMESPACE, "first"),
new Flow(NAMESPACE, "second"),
@@ -626,7 +545,7 @@ public abstract class AbstractExecutionRepositoryTest {
assertThat(result.stream().filter(executionCount -> executionCount.getFlowId().equals("third")).findFirst().get().getCount()).isEqualTo(9L);
result = executionRepository.executionCounts(
MAIN_TENANT,
tenant,
null,
null,
null,
@@ -639,14 +558,15 @@ public abstract class AbstractExecutionRepositoryTest {
@Test
protected void update() {
Execution execution = ExecutionFixture.EXECUTION_1;
executionRepository.save(ExecutionFixture.EXECUTION_1);
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Execution execution = ExecutionFixture.EXECUTION_1(tenant);
executionRepository.save(execution);
Label label = new Label("key", "value");
Execution updated = execution.toBuilder().labels(List.of(label)).build();
executionRepository.update(updated);
Optional<Execution> validation = executionRepository.findById(MAIN_TENANT, updated.getId());
Optional<Execution> validation = executionRepository.findById(tenant, updated.getId());
assertThat(validation.isPresent()).isTrue();
assertThat(validation.get().getLabels().size()).isEqualTo(1);
assertThat(validation.get().getLabels().getFirst()).isEqualTo(label);
@@ -654,13 +574,14 @@ public abstract class AbstractExecutionRepositoryTest {
@Test
void shouldFindLatestExecutionGivenState() {
Execution earliest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(10)));
Execution latest = buildWithCreatedDate(Instant.now().minus(Duration.ofMinutes(5)));
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Execution earliest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(10)));
Execution latest = buildWithCreatedDate(tenant, Instant.now().minus(Duration.ofMinutes(5)));
executionRepository.save(earliest);
executionRepository.save(latest);
Optional<Execution> result = executionRepository.findLatestForStates(MAIN_TENANT, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
Optional<Execution> result = executionRepository.findLatestForStates(tenant, "io.kestra.unittest", "full", List.of(State.Type.CREATED));
assertThat(result.isPresent()).isTrue();
assertThat(result.get().getId()).isEqualTo(latest.getId());
}
@@ -700,11 +621,11 @@ public abstract class AbstractExecutionRepositoryTest {
assertThat(data.get(0).get("date")).isEqualTo(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").format(ZonedDateTime.ofInstant(startDate, ZoneId.systemDefault()).withSecond(0).withNano(0)));
}
private static Execution buildWithCreatedDate(Instant instant) {
private static Execution buildWithCreatedDate(String tenant, Instant instant) {
return Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.flowId("full")
.flowRevision(1)
.state(new State(State.Type.CREATED, List.of(new State.History(State.Type.CREATED, instant))))
@@ -715,22 +636,24 @@ public abstract class AbstractExecutionRepositoryTest {
@Test
protected void findAllAsync() {
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant);
List<Execution> executions = executionRepository.findAllAsync(MAIN_TENANT).collectList().block();
List<Execution> executions = executionRepository.findAllAsync(tenant).collectList().block();
assertThat(executions).hasSize(29); // used by the backup so it contains TEST executions
}
@Test
protected void shouldFindByLabel() {
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant);
List<QueryFilter> filters = List.of(QueryFilter.builder()
.field(QueryFilter.Field.LABELS)
.operation(QueryFilter.Op.EQUALS)
.value(Map.of("key", "value"))
.build());
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
List<Execution> executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.size()).isEqualTo(1L);
// Filtering by two pairs of labels, since now its a and behavior, it should not return anything
@@ -739,15 +662,16 @@ public abstract class AbstractExecutionRepositoryTest {
.operation(QueryFilter.Op.EQUALS)
.value(Map.of("key", "value", "keyother", "valueother"))
.build());
executions = executionRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
executions = executionRepository.find(Pageable.from(1, 10), tenant, filters);
assertThat(executions.size()).isEqualTo(0L);
}
@Test
protected void shouldReturnLastExecutionsWhenInputsAreNull() {
inject();
var tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
inject(tenant);
List<Execution> lastExecutions = executionRepository.lastExecutions(MAIN_TENANT, null);
List<Execution> lastExecutions = executionRepository.lastExecutions(tenant, null);
assertThat(lastExecutions).isNotEmpty();
Set<String> flowIds = lastExecutions.stream().map(Execution::getFlowId).collect(Collectors.toSet());

View File

@@ -1,7 +1,6 @@
package io.kestra.core.repositories;
import com.google.common.collect.ImmutableMap;
import io.kestra.core.Helpers;
import io.kestra.core.events.CrudEvent;
import io.kestra.core.events.CrudEventType;
import io.kestra.core.exceptions.InvalidQueryFiltersException;
@@ -10,7 +9,6 @@ import io.kestra.core.models.Label;
import io.kestra.core.models.QueryFilter;
import io.kestra.core.models.QueryFilter.Field;
import io.kestra.core.models.QueryFilter.Op;
import io.kestra.core.models.SearchResult;
import io.kestra.core.models.conditions.ConditionContext;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.ExecutionTrigger;
@@ -20,7 +18,6 @@ import io.kestra.core.models.property.Property;
import io.kestra.core.models.triggers.AbstractTrigger;
import io.kestra.core.models.triggers.PollingTriggerInterface;
import io.kestra.core.models.triggers.TriggerContext;
import io.kestra.core.queues.QueueException;
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
import io.kestra.core.services.FlowService;
import io.kestra.core.utils.Await;
@@ -29,22 +26,19 @@ import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.core.debug.Return;
import io.micronaut.context.event.ApplicationEventListener;
import io.micronaut.data.model.Pageable;
import io.micronaut.data.model.Sort;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.validation.ConstraintViolationException;
import java.util.concurrent.CopyOnWriteArrayList;
import lombok.*;
import lombok.experimental.SuperBuilder;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.slf4j.event.Level;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.util.*;
@@ -52,16 +46,12 @@ import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static io.kestra.core.utils.NamespaceUtils.SYSTEM_FLOWS_DEFAULT_NAMESPACE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
// If some counts are wrong in this test it means that one of the tests is not properly deleting what it created
@KestraTest
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class AbstractFlowRepositoryTest {
public static final String TEST_TENANT_ID = "tenant";
public static final String TEST_NAMESPACE = "io.kestra.unittest";
public static final String TEST_FLOW_ID = "test";
@Inject
@@ -70,21 +60,18 @@ public abstract class AbstractFlowRepositoryTest {
@Inject
protected ExecutionRepositoryInterface executionRepository;
@Inject
private LocalFlowRepositoryLoader repositoryLoader;
@BeforeEach
protected void init() throws IOException, URISyntaxException {
TestsUtils.loads(MAIN_TENANT, repositoryLoader);
@BeforeAll
protected static void init() {
FlowListener.reset();
}
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder() {
return builder(IdUtils.create(), TEST_FLOW_ID);
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId) {
return builder(tenantId, IdUtils.create(), TEST_FLOW_ID);
}
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String flowId, String taskId) {
private static FlowWithSource.FlowWithSourceBuilder<?, ?> builder(String tenantId, String flowId, String taskId) {
return FlowWithSource.builder()
.tenantId(tenantId)
.id(flowId)
.namespace(TEST_NAMESPACE)
.tasks(Collections.singletonList(Return.builder().id(taskId).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()));
@@ -93,16 +80,16 @@ public abstract class AbstractFlowRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all(QueryFilter filter){
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = FlowWithSource.builder()
.id("filterFlowId")
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.labels(Label.from(Map.of("key", "value")))
.build();
flow = flowRepository.create(GenericFlow.of(flow));
try {
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
ArrayListTotal<Flow> entries = flowRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
assertThat(entries).hasSize(1);
} finally {
@@ -113,16 +100,16 @@ public abstract class AbstractFlowRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all_with_source(QueryFilter filter){
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = FlowWithSource.builder()
.id("filterFlowId")
.namespace(SYSTEM_FLOWS_DEFAULT_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.labels(Label.from(Map.of("key", "value")))
.build();
flow = flowRepository.create(GenericFlow.of(flow));
try {
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
ArrayListTotal<FlowWithSource> entries = flowRepository.findWithSource(Pageable.UNPAGED, tenant, List.of(filter));
assertThat(entries).hasSize(1);
} finally {
@@ -144,7 +131,7 @@ public abstract class AbstractFlowRepositoryTest {
void should_fail_to_find_all(QueryFilter filter){
assertThrows(
InvalidQueryFiltersException.class,
() -> flowRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
() -> flowRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
}
@@ -153,7 +140,7 @@ public abstract class AbstractFlowRepositoryTest {
void should_fail_to_find_all_with_source(QueryFilter filter){
assertThrows(
InvalidQueryFiltersException.class,
() -> flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
() -> flowRepository.findWithSource(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
}
@@ -176,17 +163,17 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void findById() {
FlowWithSource flow = builder()
.tenantId(MAIN_TENANT)
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = builder(tenant)
.revision(3)
.build();
flow = flowRepository.create(GenericFlow.of(flow));
try {
Optional<Flow> full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId());
Optional<Flow> full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId());
assertThat(full.isPresent()).isTrue();
assertThat(full.get().getRevision()).isEqualTo(1);
full = flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
full = flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
assertThat(full.isPresent()).isTrue();
} finally {
deleteFlow(flow);
@@ -195,17 +182,18 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void findByIdWithoutAcl() {
FlowWithSource flow = builder()
.tenantId(MAIN_TENANT)
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = builder(tenant)
.tenantId(tenant)
.revision(3)
.build();
flow = flowRepository.create(GenericFlow.of(flow));
try {
Optional<Flow> full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
Optional<Flow> full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
assertThat(full.isPresent()).isTrue();
assertThat(full.get().getRevision()).isEqualTo(1);
full = flowRepository.findByIdWithoutAcl(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.empty());
full = flowRepository.findByIdWithoutAcl(tenant, flow.getNamespace(), flow.getId(), Optional.empty());
assertThat(full.isPresent()).isTrue();
} finally {
deleteFlow(flow);
@@ -214,15 +202,16 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void findByIdWithSource() {
FlowWithSource flow = builder()
.tenantId(MAIN_TENANT)
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = builder(tenant)
.tenantId(tenant)
.revision(3)
.build();
String source = "# comment\n" + flow.sourceOrGenerateIfNull();
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, source));
flow = flowRepository.create(GenericFlow.fromYaml(tenant, source));
try {
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(MAIN_TENANT, flow.getNamespace(), flow.getId());
Optional<FlowWithSource> full = flowRepository.findByIdWithSource(tenant, flow.getNamespace(), flow.getId());
assertThat(full.isPresent()).isTrue();
full.ifPresent(current -> {
@@ -237,7 +226,8 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void save() {
FlowWithSource flow = builder().revision(12).build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = builder(tenant).revision(12).build();
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
try {
@@ -249,7 +239,8 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void saveNoRevision() {
FlowWithSource flow = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource flow = builder(tenant).build();
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
try {
@@ -260,68 +251,17 @@ public abstract class AbstractFlowRepositoryTest {
}
@Test
void findAll() {
List<Flow> save = flowRepository.findAll(MAIN_TENANT);
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void findAllWithSource() {
List<FlowWithSource> save = flowRepository.findAllWithSource(MAIN_TENANT);
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void findAllForAllTenants() {
List<Flow> save = flowRepository.findAllForAllTenants();
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void findAllWithSourceForAllTenants() {
List<FlowWithSource> save = flowRepository.findAllWithSourceForAllTenants();
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void findByNamespace() {
List<Flow> save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests");
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 24);
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
assertThat((long) save.size()).isEqualTo(1L);
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
assertThat((long) save.size()).isEqualTo(1L);
}
@Test
void findByNamespacePrefix() {
List<Flow> save = flowRepository.findByNamespacePrefix(MAIN_TENANT, "io.kestra.tests");
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests2");
assertThat((long) save.size()).isEqualTo(1L);
save = flowRepository.findByNamespace(MAIN_TENANT, "io.kestra.tests.minimal.bis");
assertThat((long) save.size()).isEqualTo(1L);
}
@Test
void findByNamespaceWithSource() {
Flow flow = builder()
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Flow flow = builder(tenant)
.revision(3)
.build();
String flowSource = "# comment\n" + flow.sourceOrGenerateIfNull();
flow = flowRepository.create(GenericFlow.fromYaml(MAIN_TENANT, flowSource));
flow = flowRepository.create(GenericFlow.fromYaml(tenant, flowSource));
try {
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(MAIN_TENANT, flow.getNamespace());
List<FlowWithSource> save = flowRepository.findByNamespaceWithSource(tenant, flow.getNamespace());
assertThat((long) save.size()).isEqualTo(1L);
assertThat(save.getFirst().getSource()).isEqualTo(FlowService.cleanupSource(flowSource));
@@ -330,175 +270,15 @@ public abstract class AbstractFlowRepositoryTest {
}
}
@Test
void findByNamespacePrefixWithSource() {
List<FlowWithSource> save = flowRepository.findByNamespacePrefixWithSource(MAIN_TENANT, "io.kestra.tests");
assertThat((long) save.size()).isEqualTo(Helpers.FLOWS_COUNT - 1);
}
@Test
void find_paginationPartial() {
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
.size())
.describedAs("When paginating at MAX-1, it should return MAX-1")
.isEqualTo(Helpers.FLOWS_COUNT - 1);
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT - 1, Sort.UNSORTED), MAIN_TENANT, null)
.size())
.describedAs("When paginating at MAX-1, it should return MAX-1")
.isEqualTo(Helpers.FLOWS_COUNT - 1);
}
@Test
void find_paginationGreaterThanExisting() {
assertThat(flowRepository.find(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
.size())
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
.isEqualTo(Helpers.FLOWS_COUNT);
assertThat(flowRepository.findWithSource(Pageable.from(1, (int) Helpers.FLOWS_COUNT + 1, Sort.UNSORTED), MAIN_TENANT, null)
.size())
.describedAs("When paginating requesting a larger amount than existing, it should return existing MAX")
.isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void find_prefixMatchingAllNamespaces() {
assertThat(flowRepository.find(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
)
).size())
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
.isEqualTo(Helpers.FLOWS_COUNT);
assertThat(flowRepository.findWithSource(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.STARTS_WITH).value("io.kestra.tests").build()
)
).size())
.describedAs("When filtering on NAMESPACE START_WITH a pattern that match all, it should return all")
.isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void find_aSpecifiedNamespace() {
assertThat(flowRepository.find(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
)
).size()).isEqualTo(1L);
assertThat(flowRepository.findWithSource(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests2").build()
)
).size()).isEqualTo(1L);
}
@Test
void find_aSpecificSubNamespace() {
assertThat(flowRepository.find(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
)
).size())
.isEqualTo(1L);
assertThat(flowRepository.findWithSource(
Pageable.UNPAGED,
MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests.minimal.bis").build()
)
).size())
.isEqualTo(1L);
}
@Test
void find_aSpecificLabel() {
assertThat(
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
)
).size())
.isEqualTo(1);
assertThat(
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("country", "FR")).build()
)
).size())
.isEqualTo(1);
}
@Test
void find_aSpecificFlowByNamespaceAndLabel() {
assertThat(
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
)
).size())
.isEqualTo(1);
assertThat(
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key2", "value2")).build()
)
).size())
.isEqualTo(1);
}
@Test
void find_noResult_forAnUnknownNamespace() {
assertThat(
flowRepository.find(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
)
).size())
.isEqualTo(0);
assertThat(
flowRepository.findWithSource(Pageable.UNPAGED, MAIN_TENANT,
List.of(
QueryFilter.builder().field(QueryFilter.Field.NAMESPACE).operation(QueryFilter.Op.EQUALS).value("io.kestra.tests").build(),
QueryFilter.builder().field(QueryFilter.Field.LABELS).operation(QueryFilter.Op.EQUALS).value(Map.of("key1", "value2")).build()
)
).size())
.isEqualTo(0);
}
@Test
protected void findSpecialChars() {
ArrayListTotal<SearchResult<Flow>> save = flowRepository.findSourceCode(Pageable.unpaged(), "https://api.chucknorris.io", MAIN_TENANT, null);
assertThat((long) save.size()).isEqualTo(2L);
}
@Test
void delete() {
Flow flow = builder().tenantId(MAIN_TENANT).build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Flow flow = builder(tenant).tenantId(tenant).build();
FlowWithSource save = flowRepository.create(GenericFlow.of(flow));
try {
assertThat(flowRepository.findById(MAIN_TENANT, save.getNamespace(), save.getId()).isPresent()).isTrue();
assertThat(flowRepository.findById(tenant, save.getNamespace(), save.getId()).isPresent()).isTrue();
} catch (Throwable e) {
deleteFlow(save);
throw e;
@@ -506,21 +286,22 @@ public abstract class AbstractFlowRepositoryTest {
Flow delete = flowRepository.delete(save);
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isFalse();
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId(), Optional.of(save.getRevision())).isPresent()).isTrue();
List<FlowWithSource> revisions = flowRepository.findRevisions(MAIN_TENANT, flow.getNamespace(), flow.getId());
List<FlowWithSource> revisions = flowRepository.findRevisions(tenant, flow.getNamespace(), flow.getId());
assertThat(revisions.getLast().getRevision()).isEqualTo(delete.getRevision());
}
@Test
void updateConflict() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String flowId = IdUtils.create();
Flow flow = Flow.builder()
.id(flowId)
.namespace(TEST_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.inputs(List.of(StringInput.builder().type(Type.STRING).id("a").build()))
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
.build();
@@ -528,12 +309,12 @@ public abstract class AbstractFlowRepositoryTest {
Flow save = flowRepository.create(GenericFlow.of(flow));
try {
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
Flow update = Flow.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest2")
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.inputs(List.of(StringInput.builder().type(Type.STRING).id("b").build()))
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
.build();
@@ -551,13 +332,14 @@ public abstract class AbstractFlowRepositoryTest {
}
@Test
void removeTrigger() throws TimeoutException, QueueException {
public void removeTrigger() throws TimeoutException {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String flowId = IdUtils.create();
Flow flow = Flow.builder()
.id(flowId)
.namespace(TEST_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.triggers(Collections.singletonList(UnitTest.builder()
.id("sleep")
.type(UnitTest.class.getName())
@@ -567,12 +349,12 @@ public abstract class AbstractFlowRepositoryTest {
flow = flowRepository.create(GenericFlow.of(flow));
try {
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
Flow update = Flow.builder()
.id(flowId)
.namespace(TEST_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.tasks(Collections.singletonList(Return.builder().id(TEST_FLOW_ID).type(Return.class.getName()).format(Property.ofValue(TEST_FLOW_ID)).build()))
.build();
;
@@ -583,21 +365,25 @@ public abstract class AbstractFlowRepositoryTest {
deleteFlow(flow);
}
Await.until(() -> FlowListener.getEmits().size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
Await.until(() -> FlowListener.filterByTenant(tenant)
.size() == 3, Duration.ofMillis(100), Duration.ofSeconds(5));
assertThat(FlowListener.filterByTenant(tenant).stream()
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(FlowListener.filterByTenant(tenant).stream()
.filter(r -> r.getType() == CrudEventType.UPDATE).count()).isEqualTo(1L);
assertThat(FlowListener.filterByTenant(tenant).stream()
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
}
@Test
void removeTriggerDelete() throws TimeoutException {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String flowId = IdUtils.create();
Flow flow = Flow.builder()
.id(flowId)
.namespace(TEST_NAMESPACE)
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.triggers(Collections.singletonList(UnitTest.builder()
.id("sleep")
.type(UnitTest.class.getName())
@@ -607,40 +393,39 @@ public abstract class AbstractFlowRepositoryTest {
Flow save = flowRepository.create(GenericFlow.of(flow));
try {
assertThat(flowRepository.findById(MAIN_TENANT, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
assertThat(flowRepository.findById(tenant, flow.getNamespace(), flow.getId()).isPresent()).isTrue();
} finally {
deleteFlow(save);
}
Await.until(() -> FlowListener.getEmits().size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(FlowListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
Await.until(() -> FlowListener.filterByTenant(tenant)
.size() == 2, Duration.ofMillis(100), Duration.ofSeconds(5));
assertThat(FlowListener.filterByTenant(tenant).stream()
.filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(FlowListener.filterByTenant(tenant).stream()
.filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
}
@Test
void findDistinctNamespace() {
List<String> distinctNamespace = flowRepository.findDistinctNamespace(MAIN_TENANT);
assertThat((long) distinctNamespace.size()).isEqualTo(9L);
}
@Test
protected void shouldReturnNullRevisionForNonExistingFlow() {
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, IdUtils.create())).isNull();
assertThat(flowRepository.lastRevision(TestsUtils.randomTenant(this.getClass().getSimpleName()), TEST_NAMESPACE, IdUtils.create())).isNull();
}
@Test
protected void shouldReturnLastRevisionOnCreate() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// When
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "???")));
Integer result = flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId);
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "???")));
Integer result = flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId);
// Then
assertThat(result).isEqualTo(1);
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
} finally {
toDelete.forEach(this::deleteFlow);
}
@@ -649,34 +434,36 @@ public abstract class AbstractFlowRepositoryTest {
@Test
protected void shouldIncrementRevisionOnDelete() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final String flowId = IdUtils.create();
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(1);
// When
flowRepository.delete(created);
// Then
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(2);
}
@Test
protected void shouldIncrementRevisionOnCreateAfterDelete() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// Given
flowRepository.delete(
flowRepository.create(createTestingLogFlow(flowId, "first"))
flowRepository.create(createTestingLogFlow(tenant, flowId, "first"))
);
// When
toDelete.add(flowRepository.create(createTestingLogFlow(flowId, "second")));
toDelete.add(flowRepository.create(createTestingLogFlow(tenant, flowId, "second")));
// Then
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(3);
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(3);
} finally {
toDelete.forEach(this::deleteFlow);
}
@@ -685,22 +472,23 @@ public abstract class AbstractFlowRepositoryTest {
@Test
protected void shouldReturnNullForLastRevisionAfterDelete() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// Given
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
toDelete.add(created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
toDelete.add(updated);
// When
flowRepository.delete(updated);
// Then
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isNull();
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isNull();
} finally {
toDelete.forEach(this::deleteFlow);
}
@@ -709,22 +497,23 @@ public abstract class AbstractFlowRepositoryTest {
@Test
protected void shouldFindAllRevisionsAfterDelete() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// Given
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
toDelete.add(created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
toDelete.add(updated);
// When
flowRepository.delete(updated);
// Then
assertThat(flowRepository.findById(TEST_TENANT_ID, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
assertThat(flowRepository.findRevisions(TEST_TENANT_ID, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
assertThat(flowRepository.findById(tenant, TEST_NAMESPACE, flowId, Optional.empty())).isEqualTo(Optional.empty());
assertThat(flowRepository.findRevisions(tenant, TEST_NAMESPACE, flowId).size()).isEqualTo(3);
} finally {
toDelete.forEach(this::deleteFlow);
}
@@ -732,21 +521,22 @@ public abstract class AbstractFlowRepositoryTest {
@Test
protected void shouldIncrementRevisionOnUpdateGivenNotEqualSource() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// Given
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
toDelete.add(created);
// When
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "second"), created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "second"), created);
toDelete.add(updated);
// Then
assertThat(updated.getRevision()).isEqualTo(2);
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(2);
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(2);
} finally {
toDelete.forEach(this::deleteFlow);
@@ -755,48 +545,39 @@ public abstract class AbstractFlowRepositoryTest {
@Test
protected void shouldNotIncrementRevisionOnUpdateGivenEqualSource() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
final List<Flow> toDelete = new ArrayList<>();
final String flowId = IdUtils.create();
try {
// Given
FlowWithSource created = flowRepository.create(createTestingLogFlow(flowId, "first"));
FlowWithSource created = flowRepository.create(createTestingLogFlow(tenant, flowId, "first"));
toDelete.add(created);
// When
FlowWithSource updated = flowRepository.update(createTestingLogFlow(flowId, "first"), created);
FlowWithSource updated = flowRepository.update(createTestingLogFlow(tenant, flowId, "first"), created);
toDelete.add(updated);
// Then
assertThat(updated.getRevision()).isEqualTo(1);
assertThat(flowRepository.lastRevision(TEST_TENANT_ID, TEST_NAMESPACE, flowId)).isEqualTo(1);
assertThat(flowRepository.lastRevision(tenant, TEST_NAMESPACE, flowId)).isEqualTo(1);
} finally {
toDelete.forEach(this::deleteFlow);
}
}
@Test
void shouldReturnForGivenQueryWildCardFilters() {
List<QueryFilter> filters = List.of(
QueryFilter.builder().field(QueryFilter.Field.QUERY).operation(QueryFilter.Op.EQUALS).value("*").build()
);
ArrayListTotal<Flow> flows = flowRepository.find(Pageable.from(1, 10), MAIN_TENANT, filters);
assertThat(flows.size()).isEqualTo(10);
assertThat(flows.getTotal()).isEqualTo(Helpers.FLOWS_COUNT);
}
@Test
void findByExecution() {
Flow flow = builder()
.tenantId(MAIN_TENANT)
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Flow flow = builder(tenant)
.revision(1)
.build();
flowRepository.create(GenericFlow.of(flow));
Execution execution = Execution.builder()
.id(IdUtils.create())
.namespace(flow.getNamespace())
.tenantId(MAIN_TENANT)
.tenantId(tenant)
.flowId(flow.getId())
.flowRevision(flow.getRevision())
.state(new State())
@@ -821,11 +602,13 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void findByExecutionNoRevision() {
Flow flow = builder()
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Flow flow = builder(tenant)
.revision(3)
.build();
flowRepository.create(GenericFlow.of(flow));
Execution execution = Execution.builder()
.tenantId(tenant)
.id(IdUtils.create())
.namespace(flow.getNamespace())
.flowId(flow.getId())
@@ -851,13 +634,14 @@ public abstract class AbstractFlowRepositoryTest {
@Test
void shouldCountForNullTenant() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
FlowWithSource toDelete = null;
try {
// Given
Flow flow = createTestFlowForNamespace(TEST_NAMESPACE);
Flow flow = createTestFlowForNamespace(tenant, TEST_NAMESPACE);
toDelete = flowRepository.create(GenericFlow.of(flow));
// When
int count = flowRepository.count(MAIN_TENANT);
int count = flowRepository.count(tenant);
// Then
Assertions.assertTrue(count > 0);
@@ -868,11 +652,11 @@ public abstract class AbstractFlowRepositoryTest {
}
}
private static Flow createTestFlowForNamespace(String namespace) {
private static Flow createTestFlowForNamespace(String tenantId, String namespace) {
return Flow.builder()
.id(IdUtils.create())
.namespace(namespace)
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.tasks(List.of(Return.builder()
.id(IdUtils.create())
.type(Return.class.getName())
@@ -891,21 +675,31 @@ public abstract class AbstractFlowRepositoryTest {
}
@Singleton
public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
@Getter
private static List<CrudEvent<Flow>> emits = new ArrayList<>();
public static class FlowListener implements ApplicationEventListener<CrudEvent<AbstractFlow>> {
private static List<CrudEvent<AbstractFlow>> emits = new CopyOnWriteArrayList<>();
@Override
public void onApplicationEvent(CrudEvent<Flow> event) {
emits.add(event);
public void onApplicationEvent(CrudEvent<AbstractFlow> event) {
//This has to be done because Micronaut may send CrudEvent<Setting> for example, and we don't want them.
if ((event.getModel() != null && event.getModel() instanceof AbstractFlow)||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof AbstractFlow)) {
emits.add(event);
}
}
public static void reset() {
emits = new ArrayList<>();
emits = new CopyOnWriteArrayList<>();
}
public static List<CrudEvent<AbstractFlow>> filterByTenant(String tenantId){
return emits.stream()
.filter(e -> (e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)) ||
(e.getModel() != null && e.getModel().getTenantId().equals(tenantId)))
.toList();
}
}
private static GenericFlow createTestingLogFlow(String id, String logMessage) {
private static GenericFlow createTestingLogFlow(String tenantId, String id, String logMessage) {
String source = """
id: %s
namespace: %s
@@ -914,7 +708,7 @@ public abstract class AbstractFlowRepositoryTest {
type: io.kestra.plugin.core.log.Log
message: %s
""".formatted(id, TEST_NAMESPACE, logMessage);
return GenericFlow.fromYaml(TEST_TENANT_ID, source);
return GenericFlow.fromYaml(tenantId, source);
}
protected static int COUNTER = 0;

View File

@@ -4,7 +4,7 @@ import io.kestra.core.models.topologies.FlowNode;
import io.kestra.core.models.topologies.FlowRelation;
import io.kestra.core.models.topologies.FlowTopology;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.tenant.TenantService;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Test;
@@ -17,21 +17,21 @@ public abstract class AbstractFlowTopologyRepositoryTest {
@Inject
private FlowTopologyRepositoryInterface flowTopologyRepository;
protected FlowTopology createSimpleFlowTopology(String flowA, String flowB, String namespace) {
protected FlowTopology createSimpleFlowTopology(String tenantId, String flowA, String flowB, String namespace) {
return FlowTopology.builder()
.relation(FlowRelation.FLOW_TASK)
.source(FlowNode.builder()
.id(flowA)
.namespace(namespace)
.tenantId(TenantService.MAIN_TENANT)
.uid(flowA)
.tenantId(tenantId)
.uid(tenantId + flowA)
.build()
)
.destination(FlowNode.builder()
.id(flowB)
.namespace(namespace)
.tenantId(TenantService.MAIN_TENANT)
.uid(flowB)
.tenantId(tenantId)
.uid(tenantId + flowB)
.build()
)
.build();
@@ -39,42 +39,45 @@ public abstract class AbstractFlowTopologyRepositoryTest {
@Test
void findByFlow() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
flowTopologyRepository.save(
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
);
List<FlowTopology> list = flowTopologyRepository.findByFlow(TenantService.MAIN_TENANT, "io.kestra.tests", "flow-a", false);
List<FlowTopology> list = flowTopologyRepository.findByFlow(tenant, "io.kestra.tests", "flow-a", false);
assertThat(list.size()).isEqualTo(1);
}
@Test
void findByNamespace() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
flowTopologyRepository.save(
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
);
flowTopologyRepository.save(
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
);
List<FlowTopology> list = flowTopologyRepository.findByNamespace(TenantService.MAIN_TENANT, "io.kestra.tests");
List<FlowTopology> list = flowTopologyRepository.findByNamespace(tenant, "io.kestra.tests");
assertThat(list.size()).isEqualTo(2);
}
@Test
void findAll() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
flowTopologyRepository.save(
createSimpleFlowTopology("flow-a", "flow-b", "io.kestra.tests")
createSimpleFlowTopology(tenant, "flow-a", "flow-b", "io.kestra.tests")
);
flowTopologyRepository.save(
createSimpleFlowTopology("flow-c", "flow-d", "io.kestra.tests")
createSimpleFlowTopology(tenant, "flow-c", "flow-d", "io.kestra.tests")
);
flowTopologyRepository.save(
createSimpleFlowTopology("flow-e", "flow-f", "io.kestra.tests.2")
createSimpleFlowTopology(tenant, "flow-e", "flow-f", "io.kestra.tests.2")
);
List<FlowTopology> list = flowTopologyRepository.findAll(TenantService.MAIN_TENANT);
List<FlowTopology> list = flowTopologyRepository.findAll(tenant);
assertThat(list.size()).isEqualTo(3);
}

View File

@@ -13,6 +13,7 @@ import io.kestra.core.models.executions.LogEntry;
import io.kestra.core.models.flows.State;
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.core.dashboard.data.Logs;
import io.micronaut.data.model.Pageable;
import jakarta.inject.Inject;
@@ -32,9 +33,7 @@ import java.util.stream.Stream;
import static io.kestra.core.models.flows.FlowScope.SYSTEM;
import static io.kestra.core.models.flows.FlowScope.USER;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatReflectiveOperationException;
import static org.junit.jupiter.api.Assertions.assertThrows;
@KestraTest
@@ -42,11 +41,11 @@ public abstract class AbstractLogRepositoryTest {
@Inject
protected LogRepositoryInterface logRepository;
protected static LogEntry.LogEntryBuilder logEntry(Level level) {
return logEntry(level, IdUtils.create());
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level) {
return logEntry(tenantId, level, IdUtils.create());
}
protected static LogEntry.LogEntryBuilder logEntry(Level level, String executionId) {
protected static LogEntry.LogEntryBuilder logEntry(String tenantId, Level level, String executionId) {
return LogEntry.builder()
.flowId("flowId")
.namespace("io.kestra.unittest")
@@ -57,7 +56,7 @@ public abstract class AbstractLogRepositoryTest {
.timestamp(Instant.now())
.level(level)
.thread("")
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.triggerId("triggerId")
.message("john doe");
}
@@ -65,9 +64,10 @@ public abstract class AbstractLogRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all(QueryFilter filter){
logRepository.save(logEntry(Level.INFO, "executionId").build());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
ArrayListTotal<LogEntry> entries = logRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
assertThat(entries).hasSize(1);
}
@@ -75,9 +75,10 @@ public abstract class AbstractLogRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_async(QueryFilter filter){
logRepository.save(logEntry(Level.INFO, "executionId").build());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
Flux<LogEntry> find = logRepository.findAsync(MAIN_TENANT, List.of(filter));
Flux<LogEntry> find = logRepository.findAsync(tenant, List.of(filter));
List<LogEntry> logEntries = find.collectList().block();
assertThat(logEntries).hasSize(1);
@@ -86,11 +87,12 @@ public abstract class AbstractLogRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_delete_with_filter(QueryFilter filter){
logRepository.save(logEntry(Level.INFO, "executionId").build());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
logRepository.save(logEntry(tenant, Level.INFO, "executionId").build());
logRepository.deleteByFilters(MAIN_TENANT, List.of(filter));
logRepository.deleteByFilters(tenant, List.of(filter));
assertThat(logRepository.findAllAsync(MAIN_TENANT).collectList().block()).isEmpty();
assertThat(logRepository.findAllAsync(tenant).collectList().block()).isEmpty();
}
@@ -150,7 +152,10 @@ public abstract class AbstractLogRepositoryTest {
void should_fail_to_find_all(QueryFilter filter){
assertThrows(
InvalidQueryFiltersException.class,
() -> logRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
() -> logRepository.find(
Pageable.UNPAGED,
TestsUtils.randomTenant(this.getClass().getSimpleName()),
List.of(filter)));
}
@@ -168,16 +173,17 @@ public abstract class AbstractLogRepositoryTest {
@Test
void all() {
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
ArrayListTotal<LogEntry> find = logRepository.find(Pageable.UNPAGED, tenant, null);
assertThat(find.size()).isZero();
LogEntry save = logRepository.save(builder.build());
logRepository.save(builder.executionKind(ExecutionKind.TEST).build()); // should only be loaded by execution id
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
find = logRepository.find(Pageable.UNPAGED, tenant, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
var filters = List.of(QueryFilter.builder()
@@ -193,7 +199,7 @@ public abstract class AbstractLogRepositoryTest {
find = logRepository.find(Pageable.UNPAGED, "doe", filters);
assertThat(find.size()).isZero();
find = logRepository.find(Pageable.UNPAGED, MAIN_TENANT, null);
find = logRepository.find(Pageable.UNPAGED, tenant, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
@@ -201,141 +207,146 @@ public abstract class AbstractLogRepositoryTest {
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
List<LogEntry> list = logRepository.findByExecutionId(MAIN_TENANT, save.getExecutionId(), null);
List<LogEntry> list = logRepository.findByExecutionId(tenant, save.getExecutionId(), null);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
list = logRepository.findByExecutionId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
list = logRepository.findByExecutionId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), null);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
list = logRepository.findByExecutionIdAndTaskId(tenant, "io.kestra.unittest", "flowId", save.getExecutionId(), save.getTaskId(), null);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
list = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null);
list = logRepository.findByExecutionIdAndTaskRunId(tenant, save.getExecutionId(), save.getTaskRunId(), null);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, save.getExecutionId(), save.getTaskRunId(), null, 0);
list = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, save.getExecutionId(), save.getTaskRunId(), null, 0);
assertThat(list.size()).isEqualTo(2);
assertThat(list.getFirst().getExecutionId()).isEqualTo(save.getExecutionId());
Integer countDeleted = logRepository.purge(Execution.builder().id(save.getExecutionId()).build());
assertThat(countDeleted).isEqualTo(2);
list = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, save.getExecutionId(), save.getTaskId(), null);
list = logRepository.findByExecutionIdAndTaskId(tenant, save.getExecutionId(), save.getTaskId(), null);
assertThat(list.size()).isZero();
}
@Test
void pageable() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String executionId = "123";
LogEntry.LogEntryBuilder builder = logEntry(Level.INFO);
LogEntry.LogEntryBuilder builder = logEntry(tenant, Level.INFO);
builder.executionId(executionId);
for (int i = 0; i < 80; i++) {
logRepository.save(builder.build());
}
builder = logEntry(Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
builder = logEntry(tenant, Level.INFO).executionId(executionId).taskId("taskId2").taskRunId("taskRunId2");
LogEntry logEntry2 = logRepository.save(builder.build());
for (int i = 0; i < 20; i++) {
logRepository.save(builder.build());
}
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(1, 50));
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(1, 50));
assertThat(find.size()).isEqualTo(50);
assertThat(find.getTotal()).isEqualTo(101L);
find = logRepository.findByExecutionId(MAIN_TENANT, executionId, null, Pageable.from(3, 50));
find = logRepository.findByExecutionId(tenant, executionId, null, Pageable.from(3, 50));
assertThat(find.size()).isEqualTo(1);
assertThat(find.getTotal()).isEqualTo(101L);
find = logRepository.findByExecutionIdAndTaskId(MAIN_TENANT, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
find = logRepository.findByExecutionIdAndTaskId(tenant, executionId, logEntry2.getTaskId(), null, Pageable.from(1, 50));
assertThat(find.size()).isEqualTo(21);
assertThat(find.getTotal()).isEqualTo(21L);
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(1, 10));
assertThat(find.size()).isEqualTo(10);
assertThat(find.getTotal()).isEqualTo(21L);
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
find = logRepository.findByExecutionIdAndTaskRunIdAndAttempt(tenant, executionId, logEntry2.getTaskRunId(), null, 0, Pageable.from(1, 10));
assertThat(find.size()).isEqualTo(10);
assertThat(find.getTotal()).isEqualTo(21L);
find = logRepository.findByExecutionIdAndTaskRunId(MAIN_TENANT, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
find = logRepository.findByExecutionIdAndTaskRunId(tenant, executionId, logEntry2.getTaskRunId(), null, Pageable.from(10, 10));
assertThat(find.size()).isZero();
}
@Test
void shouldFindByExecutionIdTestLogs() {
var builder = logEntry(Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
var builder = logEntry(tenant, Level.INFO).executionId("123").executionKind(ExecutionKind.TEST).build();
logRepository.save(builder);
List<LogEntry> logs = logRepository.findByExecutionId(MAIN_TENANT, builder.getExecutionId(), null);
List<LogEntry> logs = logRepository.findByExecutionId(tenant, builder.getExecutionId(), null);
assertThat(logs).hasSize(1);
}
@Test
void deleteByQuery() {
LogEntry log1 = logEntry(Level.INFO).build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
LogEntry log1 = logEntry(tenant, Level.INFO).build();
logRepository.save(log1);
logRepository.deleteByQuery(MAIN_TENANT, log1.getExecutionId(), null, null, null, null);
logRepository.deleteByQuery(tenant, log1.getExecutionId(), null, null, null, null);
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
ArrayListTotal<LogEntry> find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
assertThat(find.size()).isZero();
logRepository.save(log1);
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null, List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
assertThat(find.size()).isZero();
logRepository.save(log1);
logRepository.deleteByQuery(MAIN_TENANT, "io.kestra.unittest", "flowId", null);
logRepository.deleteByQuery(tenant, "io.kestra.unittest", "flowId", null);
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
assertThat(find.size()).isZero();
logRepository.save(log1);
logRepository.deleteByQuery(MAIN_TENANT, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
logRepository.deleteByQuery(tenant, null, null, log1.getExecutionId(), List.of(Level.TRACE, Level.DEBUG, Level.INFO), null, ZonedDateTime.now().plusMinutes(1));
find = logRepository.findByExecutionId(MAIN_TENANT, log1.getExecutionId(), null, Pageable.from(1, 50));
find = logRepository.findByExecutionId(tenant, log1.getExecutionId(), null, Pageable.from(1, 50));
assertThat(find.size()).isZero();
}
@Test
void findAllAsync() {
logRepository.save(logEntry(Level.INFO).build());
logRepository.save(logEntry(Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
logRepository.save(logEntry(Level.ERROR).build());
logRepository.save(logEntry(Level.WARN).build());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
logRepository.save(logEntry(tenant, Level.INFO).build());
logRepository.save(logEntry(tenant, Level.INFO).executionKind(ExecutionKind.TEST).build()); // should be present as it's used for backup
logRepository.save(logEntry(tenant, Level.ERROR).build());
logRepository.save(logEntry(tenant, Level.WARN).build());
Flux<LogEntry> find = logRepository.findAllAsync(MAIN_TENANT);
Flux<LogEntry> find = logRepository.findAllAsync(tenant);
List<LogEntry> logEntries = find.collectList().block();
assertThat(logEntries).hasSize(4);
}
@Test
void fetchData() throws IOException {
logRepository.save(logEntry(Level.INFO).build());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
logRepository.save(logEntry(tenant, Level.INFO).build());
var results = logRepository.fetchData(MAIN_TENANT,
var results = logRepository.fetchData(tenant,
Logs.builder()
.type(Logs.class.getName())
.columns(Map.of(

View File

@@ -7,6 +7,7 @@ import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.executions.metrics.MetricAggregations;
import io.kestra.core.models.executions.metrics.Timer;
import io.kestra.core.utils.TestsUtils;
import io.micronaut.data.model.Pageable;
import io.kestra.core.junit.annotations.KestraTest;
import jakarta.inject.Inject;
@@ -25,27 +26,28 @@ public abstract class AbstractMetricRepositoryTest {
@Test
void all() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String executionId = FriendlyId.createFriendlyId();
TaskRun taskRun1 = taskRun(executionId, "task");
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
MetricEntry testCounter = MetricEntry.of(taskRun1, counter("test"), ExecutionKind.TEST);
TaskRun taskRun2 = taskRun(executionId, "task");
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
metricRepository.save(counter);
metricRepository.save(testCounter); // should only be retrieved by execution id
metricRepository.save(timer);
List<MetricEntry> results = metricRepository.findByExecutionId(null, executionId, Pageable.from(1, 10));
List<MetricEntry> results = metricRepository.findByExecutionId(tenant, executionId, Pageable.from(1, 10));
assertThat(results.size()).isEqualTo(3);
results = metricRepository.findByExecutionIdAndTaskId(null, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
results = metricRepository.findByExecutionIdAndTaskId(tenant, executionId, taskRun1.getTaskId(), Pageable.from(1, 10));
assertThat(results.size()).isEqualTo(3);
results = metricRepository.findByExecutionIdAndTaskRunId(null, executionId, taskRun1.getId(), Pageable.from(1, 10));
results = metricRepository.findByExecutionIdAndTaskRunId(tenant, executionId, taskRun1.getId(), Pageable.from(1, 10));
assertThat(results.size()).isEqualTo(2);
MetricAggregations aggregationResults = metricRepository.aggregateByFlowId(
null,
tenant,
"namespace",
"flow",
null,
@@ -59,7 +61,7 @@ public abstract class AbstractMetricRepositoryTest {
assertThat(aggregationResults.getGroupBy()).isEqualTo("day");
aggregationResults = metricRepository.aggregateByFlowId(
null,
tenant,
"namespace",
"flow",
null,
@@ -76,11 +78,12 @@ public abstract class AbstractMetricRepositoryTest {
@Test
void names() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String executionId = FriendlyId.createFriendlyId();
TaskRun taskRun1 = taskRun(executionId, "task");
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
TaskRun taskRun2 = taskRun(executionId, "task2");
TaskRun taskRun2 = taskRun(tenant, executionId, "task2");
MetricEntry counter2 = MetricEntry.of(taskRun2, counter("counter2"), null);
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
@@ -90,9 +93,9 @@ public abstract class AbstractMetricRepositoryTest {
metricRepository.save(test); // should only be retrieved by execution id
List<String> flowMetricsNames = metricRepository.flowMetrics(null, "namespace", "flow");
List<String> taskMetricsNames = metricRepository.taskMetrics(null, "namespace", "flow", "task");
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(null, "namespace", "flow");
List<String> flowMetricsNames = metricRepository.flowMetrics(tenant, "namespace", "flow");
List<String> taskMetricsNames = metricRepository.taskMetrics(tenant, "namespace", "flow", "task");
List<String> tasksWithMetrics = metricRepository.tasksWithMetrics(tenant, "namespace", "flow");
assertThat(flowMetricsNames.size()).isEqualTo(2);
assertThat(taskMetricsNames.size()).isEqualTo(1);
@@ -101,17 +104,18 @@ public abstract class AbstractMetricRepositoryTest {
@Test
void findAllAsync() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
String executionId = FriendlyId.createFriendlyId();
TaskRun taskRun1 = taskRun(executionId, "task");
TaskRun taskRun1 = taskRun(tenant, executionId, "task");
MetricEntry counter = MetricEntry.of(taskRun1, counter("counter"), null);
TaskRun taskRun2 = taskRun(executionId, "task");
TaskRun taskRun2 = taskRun(tenant, executionId, "task");
MetricEntry timer = MetricEntry.of(taskRun2, timer(), null);
MetricEntry test = MetricEntry.of(taskRun2, counter("test"), ExecutionKind.TEST);
metricRepository.save(counter);
metricRepository.save(timer);
metricRepository.save(test); // should be retrieved as findAllAsync is used for backup
List<MetricEntry> results = metricRepository.findAllAsync(null).collectList().block();
List<MetricEntry> results = metricRepository.findAllAsync(tenant).collectList().block();
assertThat(results).hasSize(3);
}
@@ -123,8 +127,9 @@ public abstract class AbstractMetricRepositoryTest {
return Timer.of("counter", Duration.ofSeconds(5));
}
private TaskRun taskRun(String executionId, String taskId) {
private TaskRun taskRun(String tenantId, String executionId, String taskId) {
return TaskRun.builder()
.tenantId(tenantId)
.flowId("flow")
.namespace("namespace")
.executionId(executionId)

View File

@@ -4,6 +4,8 @@ import io.kestra.core.events.CrudEvent;
import io.kestra.core.events.CrudEventType;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.templates.Template;
import io.kestra.core.utils.Await;
import io.kestra.core.utils.TestsUtils;
import io.kestra.plugin.core.debug.Return;
import io.kestra.core.utils.IdUtils;
import io.micronaut.context.event.ApplicationEventListener;
@@ -11,7 +13,10 @@ import io.micronaut.data.model.Pageable;
import io.kestra.core.junit.annotations.KestraTest;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import org.junit.jupiter.api.BeforeEach;
import java.time.Duration;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeoutException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -20,6 +25,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.assertj.core.api.Assertions.assertThat;
@@ -28,55 +35,60 @@ public abstract class AbstractTemplateRepositoryTest {
@Inject
protected TemplateRepositoryInterface templateRepository;
@BeforeEach
protected void init() throws IOException, URISyntaxException {
@BeforeAll
protected static void init() throws IOException, URISyntaxException {
TemplateListener.reset();
}
protected static Template.TemplateBuilder<?, ?> builder() {
return builder(null);
protected static Template.TemplateBuilder<?, ?> builder(String tenantId) {
return builder(tenantId, null);
}
protected static Template.TemplateBuilder<?, ?> builder(String namespace) {
protected static Template.TemplateBuilder<?, ?> builder(String tenantId, String namespace) {
return Template.builder()
.id(IdUtils.create())
.namespace(namespace == null ? "kestra.test" : namespace)
.tenantId(tenantId)
.tasks(Collections.singletonList(Return.builder().id("test").type(Return.class.getName()).format(Property.ofValue("test")).build()));
}
@Test
void findById() {
Template template = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Template template = builder(tenant).build();
templateRepository.create(template);
Optional<Template> full = templateRepository.findById(null, template.getNamespace(), template.getId());
Optional<Template> full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
assertThat(full.isPresent()).isTrue();
assertThat(full.get().getId()).isEqualTo(template.getId());
full = templateRepository.findById(null, template.getNamespace(), template.getId());
full = templateRepository.findById(tenant, template.getNamespace(), template.getId());
assertThat(full.isPresent()).isTrue();
assertThat(full.get().getId()).isEqualTo(template.getId());
}
@Test
void findByNamespace() {
Template template1 = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Template template1 = builder(tenant).build();
Template template2 = Template.builder()
.id(IdUtils.create())
.tenantId(tenant)
.namespace("kestra.test.template").build();
templateRepository.create(template1);
templateRepository.create(template2);
List<Template> templates = templateRepository.findByNamespace(null, template1.getNamespace());
List<Template> templates = templateRepository.findByNamespace(tenant, template1.getNamespace());
assertThat(templates.size()).isGreaterThanOrEqualTo(1);
templates = templateRepository.findByNamespace(null, template2.getNamespace());
templates = templateRepository.findByNamespace(tenant, template2.getNamespace());
assertThat(templates.size()).isEqualTo(1);
}
@Test
void save() {
Template template = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Template template = builder(tenant).build();
Template save = templateRepository.create(template);
assertThat(save.getId()).isEqualTo(template.getId());
@@ -84,41 +96,42 @@ public abstract class AbstractTemplateRepositoryTest {
@Test
void findAll() {
long saveCount = templateRepository.findAll(null).size();
Template template = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
long saveCount = templateRepository.findAll(tenant).size();
Template template = builder(tenant).build();
templateRepository.create(template);
long size = templateRepository.findAll(null).size();
long size = templateRepository.findAll(tenant).size();
assertThat(size).isGreaterThan(saveCount);
templateRepository.delete(template);
assertThat((long) templateRepository.findAll(null).size()).isEqualTo(saveCount);
assertThat((long) templateRepository.findAll(tenant).size()).isEqualTo(saveCount);
}
@Test
void findAllForAllTenants() {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
long saveCount = templateRepository.findAllForAllTenants().size();
Template template = builder().build();
Template template = builder(tenant).build();
templateRepository.create(template);
long size = templateRepository.findAllForAllTenants().size();
assertThat(size).isGreaterThan(saveCount);
templateRepository.delete(template);
assertThat((long) templateRepository.findAllForAllTenants().size()).isEqualTo(saveCount);
}
@Test
void find() {
Template template1 = builder().build();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Template template1 = builder(tenant).build();
templateRepository.create(template1);
Template template2 = builder().build();
Template template2 = builder(tenant).build();
templateRepository.create(template2);
Template template3 = builder().build();
Template template3 = builder(tenant).build();
templateRepository.create(template3);
// with pageable
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, null, "kestra.test");
List<Template> save = templateRepository.find(Pageable.from(1, 10),null, tenant, "kestra.test");
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
// without pageable
save = templateRepository.find(null, null, "kestra.test");
save = templateRepository.find(null, tenant, "kestra.test");
assertThat((long) save.size()).isGreaterThanOrEqualTo(3L);
templateRepository.delete(template1);
@@ -126,31 +139,45 @@ public abstract class AbstractTemplateRepositoryTest {
templateRepository.delete(template3);
}
private static final Logger LOG = LoggerFactory.getLogger(AbstractTemplateRepositoryTest.class);
@Test
void delete() {
Template template = builder().build();
protected void delete() throws TimeoutException {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Template template = builder(tenant).build();
Template save = templateRepository.create(template);
templateRepository.delete(save);
assertThat(templateRepository.findById(null, template.getNamespace(), template.getId()).isPresent()).isFalse();
assertThat(templateRepository.findById(tenant, template.getNamespace(), template.getId()).isPresent()).isFalse();
assertThat(TemplateListener.getEmits().size()).isEqualTo(2);
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(TemplateListener.getEmits().stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
Await.until(() -> {
LOG.info("-------------> number of event: {}", TemplateListener.getEmits(tenant).size());
return TemplateListener.getEmits(tenant).size() == 2;
}, Duration.ofMillis(100), Duration.ofSeconds(5));
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.CREATE).count()).isEqualTo(1L);
assertThat(TemplateListener.getEmits(tenant).stream().filter(r -> r.getType() == CrudEventType.DELETE).count()).isEqualTo(1L);
}
@Singleton
public static class TemplateListener implements ApplicationEventListener<CrudEvent<Template>> {
private static List<CrudEvent<Template>> emits = new ArrayList<>();
private static List<CrudEvent<Template>> emits = new CopyOnWriteArrayList<>();
@Override
public void onApplicationEvent(CrudEvent<Template> event) {
emits.add(event);
//The instanceOf is required because Micronaut may send non Template event via this method
if ((event.getModel() != null && event.getModel() instanceof Template) ||
(event.getPreviousModel() != null && event.getPreviousModel() instanceof Template)) {
emits.add(event);
}
}
public static List<CrudEvent<Template>> getEmits() {
return emits;
public static List<CrudEvent<Template>> getEmits(String tenantId){
return emits.stream()
.filter(e -> (e.getModel() != null && e.getModel().getTenantId().equals(tenantId)) ||
(e.getPreviousModel() != null && e.getPreviousModel().getTenantId().equals(tenantId)))
.toList();
}
public static void reset() {

View File

@@ -9,6 +9,7 @@ import io.kestra.core.models.flows.State;
import io.kestra.core.models.triggers.Trigger;
import io.kestra.core.repositories.ExecutionRepositoryInterface.ChildFilter;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.TestsUtils;
import io.micronaut.data.model.Pageable;
import io.micronaut.data.model.Sort;
import jakarta.inject.Inject;
@@ -24,7 +25,6 @@ import java.util.Optional;
import java.util.stream.Stream;
import static io.kestra.core.models.flows.FlowScope.USER;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -35,8 +35,9 @@ public abstract class AbstractTriggerRepositoryTest {
@Inject
protected TriggerRepositoryInterface triggerRepository;
private static Trigger.TriggerBuilder<?, ?> trigger() {
private static Trigger.TriggerBuilder<?, ?> trigger(String tenantId) {
return Trigger.builder()
.tenantId(tenantId)
.flowId(IdUtils.create())
.namespace(TEST_NAMESPACE)
.triggerId(IdUtils.create())
@@ -44,9 +45,9 @@ public abstract class AbstractTriggerRepositoryTest {
.date(ZonedDateTime.now());
}
protected static Trigger generateDefaultTrigger(){
protected static Trigger generateDefaultTrigger(String tenantId){
Trigger trigger = Trigger.builder()
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.triggerId("triggerId")
.namespace("trigger.namespace")
.flowId("flowId")
@@ -59,9 +60,10 @@ public abstract class AbstractTriggerRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all(QueryFilter filter){
triggerRepository.save(generateDefaultTrigger());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
triggerRepository.save(generateDefaultTrigger(tenant));
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter));
ArrayListTotal<Trigger> entries = triggerRepository.find(Pageable.UNPAGED, tenant, List.of(filter));
assertThat(entries).hasSize(1);
}
@@ -69,9 +71,10 @@ public abstract class AbstractTriggerRepositoryTest {
@ParameterizedTest
@MethodSource("filterCombinations")
void should_find_all_async(QueryFilter filter){
triggerRepository.save(generateDefaultTrigger());
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
triggerRepository.save(generateDefaultTrigger(tenant));
List<Trigger> entries = triggerRepository.find(MAIN_TENANT, List.of(filter)).collectList().block();
List<Trigger> entries = triggerRepository.find(tenant, List.of(filter)).collectList().block();
assertThat(entries).hasSize(1);
}
@@ -92,7 +95,7 @@ public abstract class AbstractTriggerRepositoryTest {
@ParameterizedTest
@MethodSource("errorFilterCombinations")
void should_fail_to_find_all(QueryFilter filter){
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, MAIN_TENANT, List.of(filter)));
assertThrows(InvalidQueryFiltersException.class, () -> triggerRepository.find(Pageable.UNPAGED, TestsUtils.randomTenant(this.getClass().getSimpleName()), List.of(filter)));
}
static Stream<QueryFilter> errorFilterCombinations() {
@@ -110,7 +113,8 @@ public abstract class AbstractTriggerRepositoryTest {
@Test
void all() {
Trigger.TriggerBuilder<?, ?> builder = trigger();
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
Trigger.TriggerBuilder<?, ?> builder = trigger(tenant);
Optional<Trigger> findLast = triggerRepository.findLast(builder.build());
assertThat(findLast.isPresent()).isFalse();
@@ -130,47 +134,47 @@ public abstract class AbstractTriggerRepositoryTest {
assertThat(findLast.get().getExecutionId()).isEqualTo(save.getExecutionId());
triggerRepository.save(trigger().build());
triggerRepository.save(trigger().build());
Trigger searchedTrigger = trigger().build();
triggerRepository.save(trigger(tenant).build());
triggerRepository.save(trigger(tenant).build());
Trigger searchedTrigger = trigger(tenant).build();
triggerRepository.save(searchedTrigger);
List<Trigger> all = triggerRepository.findAllForAllTenants();
assertThat(all.size()).isEqualTo(4);
assertThat(all.size()).isGreaterThanOrEqualTo(4);
all = triggerRepository.findAll(null);
all = triggerRepository.findAll(tenant);
assertThat(all.size()).isEqualTo(4);
String namespacePrefix = "io.kestra.another";
String namespace = namespacePrefix + ".ns";
Trigger trigger = trigger().namespace(namespace).build();
Trigger trigger = trigger(tenant).namespace(namespace).build();
triggerRepository.save(trigger);
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, null, null);
List<Trigger> find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, null, null);
assertThat(find.size()).isEqualTo(4);
assertThat(find.getFirst().getNamespace()).isEqualTo(namespace);
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, null, null, searchedTrigger.getFlowId(), null);
find = triggerRepository.find(Pageable.from(1, 4, Sort.of(Sort.Order.asc("namespace"))), null, tenant, null, searchedTrigger.getFlowId(), null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getFlowId()).isEqualTo(searchedTrigger.getFlowId());
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, null, namespacePrefix, null, null);
find = triggerRepository.find(Pageable.from(1, 100, Sort.of(Sort.Order.asc(triggerRepository.sortMapping().apply("triggerId")))), null, tenant, namespacePrefix, null, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
// Full text search is on namespace, flowId, triggerId, executionId
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), null, null, null, null);
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), trigger.getNamespace(), tenant, null, null, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getTriggerId()).isEqualTo(trigger.getTriggerId());
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), null, null, null, null);
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getFlowId(), tenant, null, null, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), null, null, null, null);
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getTriggerId(), tenant, null, null, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), null, null, null, null);
find = triggerRepository.find(Pageable.from(1, 100, Sort.UNSORTED), searchedTrigger.getExecutionId(), tenant, null, null, null);
assertThat(find.size()).isEqualTo(1);
assertThat(find.getFirst().getTriggerId()).isEqualTo(searchedTrigger.getTriggerId());
}
@@ -178,15 +182,17 @@ public abstract class AbstractTriggerRepositoryTest {
@Test
void shouldCountForNullTenant() {
// Given
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
triggerRepository.save(Trigger
.builder()
.tenantId(tenant)
.triggerId(IdUtils.create())
.flowId(IdUtils.create())
.namespace("io.kestra.unittest")
.build()
);
// When
int count = triggerRepository.count(null);
int count = triggerRepository.count(tenant);
// Then
assertThat(count).isEqualTo(1);
}

View File

@@ -1,88 +1,92 @@
package io.kestra.core.repositories;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import com.google.common.collect.ImmutableMap;
import io.kestra.core.models.executions.*;
import io.kestra.core.models.flows.State;
import io.kestra.core.utils.IdUtils;
import java.time.Duration;
import java.util.Collections;
class ExecutionFixture {
public static final Execution EXECUTION_1 = Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(MAIN_TENANT)
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", "value"))
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", "value"
)))
.build()
))
.build();
public static Execution EXECUTION_1(String tenant) {
return Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(tenant)
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", "value"))
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", "value"
)))
.build()
))
.build();
}
public static final Execution EXECUTION_2 = Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(MAIN_TENANT)
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", 1))
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", 1
)))
.build()
))
.build();
public static Execution EXECUTION_2(String tenant) {
return Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(tenant)
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", 1))
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", 1
)))
.build()
))
.build();
}
public static final Execution EXECUTION_TEST = Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", 1))
.kind(ExecutionKind.TEST)
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", 1
)))
.build()
))
.build();
}
public static Execution EXECUTION_TEST(String tenant) {
return Execution.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tenantId(tenant)
.flowId("full")
.flowRevision(1)
.state(new State())
.inputs(ImmutableMap.of("test", 1))
.kind(ExecutionKind.TEST)
.taskRunList(Collections.singletonList(
TaskRun.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.flowId("full")
.state(new State())
.attempts(Collections.singletonList(
TaskRunAttempt.builder()
.build()
))
.outputs(Variables.inMemory(ImmutableMap.of(
"out", 1
)))
.build()
))
.build();
}
}

View File

@@ -1,10 +1,6 @@
package io.kestra.core.runners;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.executions.Execution;
@@ -13,36 +9,36 @@ import io.kestra.core.models.flows.State;
import io.kestra.core.queues.QueueException;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.plugin.core.flow.EachSequentialTest;
import io.kestra.plugin.core.flow.FlowCaseTest;
import io.kestra.plugin.core.flow.ForEachItemCaseTest;
import io.kestra.plugin.core.flow.PauseTest;
import io.kestra.plugin.core.flow.LoopUntilCaseTest;
import io.kestra.plugin.core.flow.WorkingDirectoryTest;
import io.kestra.plugin.core.flow.*;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junitpioneer.jupiter.RetryingTest;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest(startRunner = true)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
//@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
// must be per-class to allow calling once init() which took a lot of time
public abstract class AbstractRunnerTest {
public static final String TENANT_1 = "tenant1";
public static final String TENANT_2 = "tenant2";
@Inject
protected RunnerUtils runnerUtils;
protected TestRunnerUtils runnerUtils;
@Inject
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
protected QueueInterface<LogEntry> logsQueue;
@Inject
private RestartCaseTest restartCaseTest;
protected RestartCaseTest restartCaseTest;
@Inject
protected FlowTriggerCaseTest flowTriggerCaseTest;
@@ -54,13 +50,13 @@ public abstract class AbstractRunnerTest {
private PluginDefaultsCaseTest pluginDefaultsCaseTest;
@Inject
private FlowCaseTest flowCaseTest;
protected FlowCaseTest flowCaseTest;
@Inject
private WorkingDirectoryTest.Suite workingDirectoryTest;
@Inject
private PauseTest.Suite pauseTest;
protected PauseTest.Suite pauseTest;
@Inject
private SkipExecutionCaseTest skipExecutionCaseTest;
@@ -72,10 +68,10 @@ public abstract class AbstractRunnerTest {
protected LoopUntilCaseTest loopUntilTestCaseTest;
@Inject
private FlowConcurrencyCaseTest flowConcurrencyCaseTest;
protected FlowConcurrencyCaseTest flowConcurrencyCaseTest;
@Inject
private ScheduleDateCaseTest scheduleDateCaseTest;
protected ScheduleDateCaseTest scheduleDateCaseTest;
@Inject
protected FlowInputOutput flowIO;
@@ -84,7 +80,7 @@ public abstract class AbstractRunnerTest {
private SLATestCase slaTestCase;
@Inject
private ChangeStateTestCase changeStateTestCase;
protected ChangeStateTestCase changeStateTestCase;
@Inject
private AfterExecutionTestCase afterExecutionTestCase;
@@ -115,7 +111,7 @@ public abstract class AbstractRunnerTest {
assertThat(execution.getTaskRunList()).hasSize(8);
}
@RetryingTest(5)
@Test
@ExecuteFlow("flows/valids/parallel-nested.yaml")
void parallelNested(Execution execution) {
assertThat(execution.getTaskRunList()).hasSize(11);
@@ -157,27 +153,27 @@ public abstract class AbstractRunnerTest {
restartCaseTest.restartFailedThenSuccess();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/restart-each.yaml"})
void replay() throws Exception {
restartCaseTest.replay();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/failed-first.yaml"})
void restartMultiple() throws Exception {
restartCaseTest.restartMultiple();
}
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
@Test
@LoadFlows({"flows/valids/restart_always_failed.yaml"})
void restartFailedThenFailureWithGlobalErrors() throws Exception {
restartCaseTest.restartFailedThenFailureWithGlobalErrors();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/restart_local_errors.yaml"})
void restartFailedThenFailureWithLocalErrors() throws Exception {
protected void restartFailedThenFailureWithLocalErrors() throws Exception {
restartCaseTest.restartFailedThenFailureWithLocalErrors();
}
@@ -199,29 +195,27 @@ public abstract class AbstractRunnerTest {
restartCaseTest.restartFailedWithAfterExecution();
}
@RetryingTest(5)
@LoadFlows({"flows/valids/trigger-flow-listener-no-inputs.yaml",
@Test
@LoadFlows(value = {"flows/valids/trigger-flow-listener-no-inputs.yaml",
"flows/valids/trigger-flow-listener.yaml",
"flows/valids/trigger-flow-listener-namespace-condition.yaml",
"flows/valids/trigger-flow.yaml"})
"flows/valids/trigger-flow.yaml"}, tenantId = "listener-tenant")
void flowTrigger() throws Exception {
flowTriggerCaseTest.trigger();
flowTriggerCaseTest.trigger("listener-tenant");
}
@RetryingTest(5) // flaky on CI but never fail locally
@Test // flaky on CI but never fail locally
@LoadFlows({"flows/valids/trigger-flow-listener-with-pause.yaml",
"flows/valids/trigger-flow-with-pause.yaml"})
void flowTriggerWithPause() throws Exception {
flowTriggerCaseTest.triggerWithPause();
}
@FlakyTest
@Disabled
@Test
@LoadFlows({"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
"flows/valids/trigger-flow-with-concurrency-limit.yaml"})
@LoadFlows(value = {"flows/valids/trigger-flow-listener-with-concurrency-limit.yaml",
"flows/valids/trigger-flow-with-concurrency-limit.yaml"}, tenantId = "trigger-tenant")
void flowTriggerWithConcurrencyLimit() throws Exception {
flowTriggerCaseTest.triggerWithConcurrencyLimit();
flowTriggerCaseTest.triggerWithConcurrencyLimit("trigger-tenant");
}
@Test
@@ -232,12 +226,12 @@ public abstract class AbstractRunnerTest {
multipleConditionTriggerCaseTest.trigger();
}
@RetryingTest(5) // Flaky on CI but never locally even with 100 repetitions
@LoadFlows({"flows/valids/trigger-flow-listener-namespace-condition.yaml",
@Test // Flaky on CI but never locally even with 100 repetitions
@LoadFlows(value = {"flows/valids/trigger-flow-listener-namespace-condition.yaml",
"flows/valids/trigger-multiplecondition-flow-c.yaml",
"flows/valids/trigger-multiplecondition-flow-d.yaml"})
"flows/valids/trigger-multiplecondition-flow-d.yaml"}, tenantId = "condition-tenant")
void multipleConditionTriggerFailed() throws Exception {
multipleConditionTriggerCaseTest.failed();
multipleConditionTriggerCaseTest.failed("condition-tenant");
}
@Test
@@ -248,12 +242,13 @@ public abstract class AbstractRunnerTest {
multipleConditionTriggerCaseTest.flowTriggerPreconditions();
}
@Disabled
@Test
@LoadFlows({"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
@LoadFlows(value = {"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
"flows/valids/flow-trigger-preconditions-flow-a.yaml",
"flows/valids/flow-trigger-preconditions-flow-b.yaml"})
"flows/valids/flow-trigger-preconditions-flow-b.yaml"}, tenantId = TENANT_1)
void flowTriggerPreconditionsMergeOutputs() throws Exception {
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs();
multipleConditionTriggerCaseTest.flowTriggerPreconditionsMergeOutputs(TENANT_1);
}
@Test
@@ -262,7 +257,7 @@ public abstract class AbstractRunnerTest {
multipleConditionTriggerCaseTest.flowTriggerOnPaused();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/each-null.yaml"})
void eachWithNull() throws Exception {
EachSequentialTest.eachNullTest(runnerUtils, logsQueue);
@@ -274,7 +269,7 @@ public abstract class AbstractRunnerTest {
pluginDefaultsCaseTest.taskDefaults();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/switch.yaml",
"flows/valids/task-flow.yaml",
"flows/valids/task-flow-inherited-labels.yaml"})
@@ -283,19 +278,19 @@ public abstract class AbstractRunnerTest {
}
@Test
@LoadFlows({"flows/valids/switch.yaml",
@LoadFlows(value = {"flows/valids/switch.yaml",
"flows/valids/task-flow.yaml",
"flows/valids/task-flow-inherited-labels.yaml"})
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_1)
void flowWaitFailed() throws Exception {
flowCaseTest.waitFailed();
flowCaseTest.waitFailed(TENANT_1);
}
@Test
@LoadFlows({"flows/valids/switch.yaml",
@LoadFlows(value = {"flows/valids/switch.yaml",
"flows/valids/task-flow.yaml",
"flows/valids/task-flow-inherited-labels.yaml"})
"flows/valids/task-flow-inherited-labels.yaml"}, tenantId = TENANT_2)
public void invalidOutputs() throws Exception {
flowCaseTest.invalidOutputs();
flowCaseTest.invalidOutputs(TENANT_2);
}
@Test
@@ -305,9 +300,9 @@ public abstract class AbstractRunnerTest {
}
@Test
@LoadFlows({"flows/valids/working-directory.yaml"})
@LoadFlows(value = {"flows/valids/working-directory.yaml"}, tenantId = TENANT_1)
public void workerFailed() throws Exception {
workingDirectoryTest.failed(runnerUtils);
workingDirectoryTest.failed(TENANT_1, runnerUtils);
}
@Test
@@ -322,7 +317,7 @@ public abstract class AbstractRunnerTest {
workingDirectoryTest.cache(runnerUtils);
}
@RetryingTest(5) // flaky on MySQL
@Test // flaky on MySQL
@LoadFlows({"flows/valids/pause-test.yaml"})
public void pauseRun() throws Exception {
pauseTest.run(runnerUtils);
@@ -358,44 +353,44 @@ public abstract class AbstractRunnerTest {
skipExecutionCaseTest.skipExecution();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
"flows/valids/for-each-item.yaml"})
protected void forEachItem() throws Exception {
forEachItemCaseTest.forEachItem();
}
@RetryingTest(5)
@LoadFlows({"flows/valids/for-each-item.yaml"})
@Test
@LoadFlows(value = {"flows/valids/for-each-item.yaml"}, tenantId = TENANT_1)
protected void forEachItemEmptyItems() throws Exception {
forEachItemCaseTest.forEachItemEmptyItems();
forEachItemCaseTest.forEachItemEmptyItems(TENANT_1);
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/for-each-item-subflow-failed.yaml",
"flows/valids/for-each-item-failed.yaml"})
protected void forEachItemFailed() throws Exception {
forEachItemCaseTest.forEachItemFailed();
}
@RetryingTest(5)
@Test
@LoadFlows({"flows/valids/for-each-item-outputs-subflow.yaml",
"flows/valids/for-each-item-outputs.yaml"})
protected void forEachItemSubflowOutputs() throws Exception {
forEachItemCaseTest.forEachItemWithSubflowOutputs();
}
@RetryingTest(5) // flaky on CI but always pass locally even with 100 iterations
@LoadFlows({"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"})
@Test // flaky on CI but always pass locally even with 100 iterations
@LoadFlows(value = {"flows/valids/restart-for-each-item.yaml", "flows/valids/restart-child.yaml"}, tenantId = TENANT_1)
void restartForEachItem() throws Exception {
forEachItemCaseTest.restartForEachItem();
forEachItemCaseTest.restartForEachItem(TENANT_1);
}
@RetryingTest(5)
@LoadFlows({"flows/valids/for-each-item-subflow.yaml",
"flows/valids/for-each-item-in-if.yaml"})
@Test
@LoadFlows(value = {"flows/valids/for-each-item-subflow.yaml",
"flows/valids/for-each-item-in-if.yaml"}, tenantId = TENANT_1)
protected void forEachItemInIf() throws Exception {
forEachItemCaseTest.forEachItemInIf();
forEachItemCaseTest.forEachItemInIf(TENANT_1);
}
@Test
@@ -436,9 +431,9 @@ public abstract class AbstractRunnerTest {
}
@Test
@LoadFlows({"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"})
@LoadFlows(value = {"flows/valids/flow-concurrency-for-each-item.yaml", "flows/valids/flow-concurrency-queue.yml"}, tenantId = TENANT_1)
protected void flowConcurrencyWithForEachItem() throws Exception {
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem();
flowConcurrencyCaseTest.flowConcurrencyWithForEachItem(TENANT_1);
}
@Test
@@ -453,6 +448,12 @@ public abstract class AbstractRunnerTest {
flowConcurrencyCaseTest.flowConcurrencyQueueAfterExecution();
}
@Test
@LoadFlows(value = {"flows/valids/flow-concurrency-subflow.yml", "flows/valids/flow-concurrency-cancel.yml"}, tenantId = TENANT_1)
void flowConcurrencySubflow() throws Exception {
flowConcurrencyCaseTest.flowConcurrencySubflow(TENANT_1);
}
@Test
@ExecuteFlow("flows/valids/executable-fail.yml")
void badExecutable(Execution execution) {
@@ -505,9 +506,9 @@ public abstract class AbstractRunnerTest {
}
@Test
@LoadFlows({"flows/valids/minimal.yaml"})
@LoadFlows(value = {"flows/valids/minimal.yaml"}, tenantId = TENANT_1)
void shouldScheduleOnDate() throws Exception {
scheduleDateCaseTest.shouldScheduleOnDate();
scheduleDateCaseTest.shouldScheduleOnDate(TENANT_1);
}
@Test
@@ -529,15 +530,15 @@ public abstract class AbstractRunnerTest {
}
@Test
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_1)
void executionConditionSLAShouldCancel() throws Exception {
slaTestCase.executionConditionSLAShouldCancel();
slaTestCase.executionConditionSLAShouldCancel(TENANT_1);
}
@Test
@LoadFlows({"flows/valids/sla-execution-condition.yaml"})
@LoadFlows(value = {"flows/valids/sla-execution-condition.yaml"}, tenantId = TENANT_2)
void executionConditionSLAShouldLabel() throws Exception {
slaTestCase.executionConditionSLAShouldLabel();
slaTestCase.executionConditionSLAShouldLabel(TENANT_2);
}
@Test
@@ -557,15 +558,15 @@ public abstract class AbstractRunnerTest {
}
@Test
@ExecuteFlow("flows/valids/failed-first.yaml")
@ExecuteFlow(value = "flows/valids/failed-first.yaml", tenantId = TENANT_1)
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
changeStateTestCase.changeStateShouldEndsInSuccess(execution);
}
@Test
@LoadFlows({"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"})
@LoadFlows(value = {"flows/valids/failed-first.yaml", "flows/valids/subflow-parent-of-failed.yaml"}, tenantId = TENANT_2)
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess();
changeStateTestCase.changeStateInSubflowShouldEndsParentFlowInSuccess(TENANT_2);
}
@Test

View File

@@ -3,25 +3,18 @@ package io.kestra.core.runners;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.services.ExecutionService;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import reactor.core.publisher.Flux;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
@Singleton
public class ChangeStateTestCase {
public static final String NAMESPACE = "io.kestra.tests";
@Inject
private FlowRepositoryInterface flowRepository;
@@ -29,11 +22,7 @@ public class ChangeStateTestCase {
private ExecutionService executionService;
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
private QueueInterface<Execution> executionQueue;
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
public void changeStateShouldEndsInSuccess(Execution execution) throws Exception {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
@@ -41,73 +30,40 @@ public class ChangeStateTestCase {
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
// await for the last execution
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Execution> lastExecution = new AtomicReference<>();
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
Execution exec = either.getLeft();
if (execution.getId().equals(exec.getId()) && exec.getState().getCurrent() == State.Type.SUCCESS) {
lastExecution.set(exec);
latch.countDown();
}
});
Flow flow = flowRepository.findByExecution(execution);
Execution markedAs = executionService.markAs(execution, flow, execution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
executionQueue.emit(markedAs);
Execution lastExecution = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), markedAs);
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
receivedExecutions.blockLast();
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(lastExecution.get().getTaskRunList()).hasSize(2);
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(lastExecution.getTaskRunList()).hasSize(2);
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
public void changeStateInSubflowShouldEndsParentFlowInSuccess() throws Exception {
// await for the subflow execution
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Execution> lastExecution = new AtomicReference<>();
Flux<Execution> receivedExecutions = TestsUtils.receive(executionQueue, either -> {
Execution exec = either.getLeft();
if ("failed-first".equals(exec.getFlowId()) && exec.getState().getCurrent() == State.Type.FAILED) {
lastExecution.set(exec);
latch.countDown();
}
});
public void changeStateInSubflowShouldEndsParentFlowInSuccess(String tenantId) throws Exception {
// run the parent flow
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "subflow-parent-of-failed");
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "subflow-parent-of-failed");
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
assertThat(execution.getTaskRunList()).hasSize(1);
assertThat(execution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
// assert on the subflow
assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
receivedExecutions.blockLast();
assertThat(lastExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED);
assertThat(lastExecution.get().getTaskRunList()).hasSize(1);
assertThat(lastExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
// await for the parent execution
CountDownLatch parentLatch = new CountDownLatch(1);
AtomicReference<Execution> lastParentExecution = new AtomicReference<>();
receivedExecutions = TestsUtils.receive(executionQueue, either -> {
Execution exec = either.getLeft();
if (execution.getId().equals(exec.getId()) && exec.getState().isTerminated()) {
lastParentExecution.set(exec);
parentLatch.countDown();
}
});
Execution lastExecution = runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.FAILED), tenantId, NAMESPACE, "failed-first");
assertThat(lastExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
assertThat(lastExecution.getTaskRunList()).hasSize(1);
assertThat(lastExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
// restart the subflow
Flow flow = flowRepository.findByExecution(lastExecution.get());
Execution markedAs = executionService.markAs(lastExecution.get(), flow, lastExecution.get().getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
executionQueue.emit(markedAs);
Flow flow = flowRepository.findByExecution(lastExecution);
Execution markedAs = executionService.markAs(lastExecution, flow, lastExecution.getTaskRunList().getFirst().getId(), State.Type.SUCCESS);
runnerUtils.emitAndAwaitExecution(e -> e.getState().isTerminated(), markedAs);
//We wait for the subflow execution to pass from failed to success
Execution lastParentExecution = runnerUtils.awaitFlowExecution(e ->
e.getTaskRunList().getFirst().getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "subflow-parent-of-failed");
// assert for the parent flow
assertThat(parentLatch.await(10, TimeUnit.SECONDS)).isTrue();
receivedExecutions.blockLast();
assertThat(lastParentExecution.get().getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
assertThat(lastParentExecution.get().getTaskRunList()).hasSize(1);
assertThat(lastParentExecution.get().getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(lastParentExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED); // FIXME should be success but it's FAILED on unit tests
assertThat(lastParentExecution.getTaskRunList()).hasSize(1);
assertThat(lastParentExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
}

View File

@@ -18,7 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class EmptyVariablesTest {
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
@Inject
private FlowInputOutput flowIO;

View File

@@ -20,6 +20,7 @@ import io.kestra.plugin.core.debug.Return;
import io.kestra.plugin.core.flow.Pause;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.RetryingTest;
import org.slf4j.event.Level;
@@ -40,6 +41,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
@Slf4j
@KestraTest(startRunner = true)
class ExecutionServiceTest {
public static final String TENANT_1 = "tenant1";
public static final String TENANT_2 = "tenant2";
public static final String TENANT_3 = "tenant3";
@Inject
ExecutionService executionService;
@@ -53,7 +58,7 @@ class ExecutionServiceTest {
LogRepositoryInterface logRepository;
@Inject
RunnerUtils runnerUtils;
TestRunnerUtils runnerUtils;
@Test
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
@@ -75,13 +80,13 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/restart_last_failed.yaml"})
@LoadFlows(value = {"flows/valids/restart_last_failed.yaml"}, tenantId = TENANT_1)
void restartSimpleRevision() throws Exception {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart_last_failed");
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart_last_failed");
assertThat(execution.getTaskRunList()).hasSize(3);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
FlowWithSource flow = flowRepository.findByIdWithSource(MAIN_TENANT, "io.kestra.tests", "restart_last_failed").orElseThrow();
FlowWithSource flow = flowRepository.findByIdWithSource(TENANT_1, "io.kestra.tests", "restart_last_failed").orElseThrow();
flowRepository.update(
GenericFlow.of(flow),
flow.updateTask(
@@ -124,9 +129,9 @@ class ExecutionServiceTest {
}
@RetryingTest(5)
@LoadFlows({"flows/valids/restart-each.yaml"})
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_1)
void restartFlowable2() throws Exception {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "SECOND"));
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
Execution restart = executionService.restart(execution, null);
@@ -177,9 +182,9 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/logs.yaml"})
@LoadFlows(value = {"flows/valids/logs.yaml"}, tenantId = TENANT_1)
void replaySimple() throws Exception {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "logs");
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "logs");
assertThat(execution.getTaskRunList()).hasSize(5);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
@@ -196,9 +201,9 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/restart-each.yaml"})
@LoadFlows(value = {"flows/valids/restart-each.yaml"}, tenantId = TENANT_2)
void replayFlowable() throws Exception {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
Execution execution = runnerUtils.runOne(TENANT_2, "io.kestra.tests", "restart-each", null, (f, e) -> ImmutableMap.of("failed", "NO"));
assertThat(execution.getTaskRunList()).hasSize(20);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
@@ -213,6 +218,7 @@ class ExecutionServiceTest {
assertThat(restart.getLabels()).contains(new Label(Label.REPLAY, "true"));
}
@Disabled
@Test
@LoadFlows({"flows/valids/parallel-nested.yaml"})
void replayParallel() throws Exception {
@@ -234,7 +240,7 @@ class ExecutionServiceTest {
}
@Test
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_2)
void replayEachSeq(Execution execution) throws Exception {
assertThat(execution.getTaskRunList()).hasSize(23);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
@@ -253,7 +259,7 @@ class ExecutionServiceTest {
}
@Test
@ExecuteFlow("flows/valids/each-sequential-nested.yaml")
@ExecuteFlow(value = "flows/valids/each-sequential-nested.yaml", tenantId = TENANT_1)
void replayEachSeq2(Execution execution) throws Exception {
assertThat(execution.getTaskRunList()).hasSize(23);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
@@ -312,9 +318,9 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/each-parallel-nested.yaml"})
@LoadFlows(value = {"flows/valids/each-parallel-nested.yaml"}, tenantId = TENANT_1)
void markAsEachPara() throws Exception {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "each-parallel-nested");
Execution execution = runnerUtils.runOne(TENANT_1, "io.kestra.tests", "each-parallel-nested");
Flow flow = flowRepository.findByExecution(execution);
assertThat(execution.getTaskRunList()).hasSize(11);
@@ -364,9 +370,9 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/pause-test.yaml"})
@LoadFlows(value = {"flows/valids/pause-test.yaml"}, tenantId = TENANT_1)
void resumePausedToKilling() throws Exception {
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause-test");
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause-test");
Flow flow = flowRepository.findByExecution(execution);
assertThat(execution.getTaskRunList()).hasSize(1);
@@ -379,7 +385,7 @@ class ExecutionServiceTest {
}
@Test
@ExecuteFlow("flows/valids/logs.yaml")
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_2)
void deleteExecution(Execution execution) throws IOException, TimeoutException {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
@@ -391,7 +397,7 @@ class ExecutionServiceTest {
}
@Test
@ExecuteFlow("flows/valids/logs.yaml")
@ExecuteFlow(value = "flows/valids/logs.yaml", tenantId = TENANT_3)
void deleteExecutionKeepLogs(Execution execution) throws IOException, TimeoutException {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
Await.until(() -> logRepository.findByExecutionId(execution.getTenantId(), execution.getId(), Level.TRACE).size() == 5, Duration.ofMillis(10), Duration.ofSeconds(5));
@@ -431,9 +437,9 @@ class ExecutionServiceTest {
}
@Test
@LoadFlows({"flows/valids/pause_no_tasks.yaml"})
@LoadFlows(value = {"flows/valids/pause_no_tasks.yaml"}, tenantId = TENANT_1)
void killToState() throws Exception {
Execution execution = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "pause_no_tasks");
Execution execution = runnerUtils.runOneUntilPaused(TENANT_1, "io.kestra.tests", "pause_no_tasks");
Flow flow = flowRepository.findByExecution(execution);
Execution killed = executionService.kill(execution, flow, Optional.of(State.Type.CANCELLED));

View File

@@ -18,11 +18,14 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest(rebuildContext = true)
@Execution(ExecutionMode.SAME_THREAD)
class FilesServiceTest {
@Inject
private TestRunContextFactory runContextFactory;

View File

@@ -3,19 +3,15 @@ package io.kestra.core.runners;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.flows.State.History;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.queues.QueueException;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.services.ExecutionService;
import io.kestra.core.storages.StorageInterface;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import reactor.core.publisher.Flux;
import java.io.File;
import java.io.FileInputStream;
@@ -25,24 +21,21 @@ import java.net.URISyntaxException;
import java.nio.file.Files;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.IntStream;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Singleton
public class FlowConcurrencyCaseTest {
public static final String NAMESPACE = "io.kestra.tests";
@Inject
private StorageInterface storageInterface;
@Inject
protected RunnerUtils runnerUtils;
protected TestRunnerUtils runnerUtils;
@Inject
private FlowInputOutput flowIO;
@@ -50,354 +43,168 @@ public class FlowConcurrencyCaseTest {
@Inject
private FlowRepositoryInterface flowRepository;
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface<Execution> executionQueue;
@Inject
private ExecutionService executionService;
public void flowConcurrencyCancel() throws TimeoutException, QueueException, InterruptedException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel");
public void flowConcurrencyCancel() throws TimeoutException, QueueException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel", null, null, Duration.ofSeconds(30));
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel");
assertThat(execution1.getState().isRunning()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
CountDownLatch latch1 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution1.getId())) {
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch1.countDown();
}
}
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
});
assertTrue(latch1.await(1, TimeUnit.MINUTES));
receive.blockLast();
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
}
public void flowConcurrencyFail() throws TimeoutException, QueueException, InterruptedException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-fail");
public void flowConcurrencyFail() throws TimeoutException, QueueException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail", null, null, Duration.ofSeconds(30));
Execution execution2 = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "flow-concurrency-fail");
assertThat(execution1.getState().isRunning()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.FAILED);
CountDownLatch latch1 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution1.getId())) {
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch1.countDown();
}
}
// FIXME we should fail if we receive the cancel execution again but on Kafka it happens
});
assertTrue(latch1.await(1, TimeUnit.MINUTES));
receive.blockLast();
runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
}
public void flowConcurrencyQueue() throws TimeoutException, QueueException, InterruptedException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
public void flowConcurrencyQueue() throws QueueException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", null, null, Duration.ofSeconds(30));
Flow flow = flowRepository
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue", Optional.empty())
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue", Optional.empty())
.orElseThrow();
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
executionQueue.emit(execution2);
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
assertThat(execution1.getState().isRunning()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
var executionResult1 = new AtomicReference<Execution>();
var executionResult2 = new AtomicReference<Execution>();
CountDownLatch latch1 = new CountDownLatch(1);
CountDownLatch latch2 = new CountDownLatch(1);
CountDownLatch latch3 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution1.getId())) {
executionResult1.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch1.countDown();
}
}
if (e.getLeft().getId().equals(execution2.getId())) {
executionResult2.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
latch2.countDown();
}
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch3.countDown();
}
}
});
assertTrue(latch1.await(1, TimeUnit.MINUTES));
assertTrue(latch2.await(1, TimeUnit.MINUTES));
assertTrue(latch3.await(1, TimeUnit.MINUTES));
receive.blockLast();
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
}
public void flowConcurrencyQueuePause() throws TimeoutException, QueueException, InterruptedException {
AtomicReference<String> firstExecutionId = new AtomicReference<>();
var firstExecutionResult = new AtomicReference<Execution>();
var secondExecutionResult = new AtomicReference<Execution>();
CountDownLatch firstExecutionLatch = new CountDownLatch(1);
CountDownLatch secondExecutionLatch = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (!"flow-concurrency-queue-pause".equals(e.getLeft().getFlowId())){
return;
}
String currentId = e.getLeft().getId();
Type currentState = e.getLeft().getState().getCurrent();
if (firstExecutionId.get() == null) {
firstExecutionId.set(currentId);
}
if (currentId.equals(firstExecutionId.get())) {
if (currentState == State.Type.SUCCESS) {
firstExecutionResult.set(e.getLeft());
firstExecutionLatch.countDown();
}
} else {
if (currentState == State.Type.SUCCESS) {
secondExecutionResult.set(e.getLeft());
secondExecutionLatch.countDown();
}
}
});
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause");
public void flowConcurrencyQueuePause() throws QueueException {
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause");
Flow flow = flowRepository
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-pause", Optional.empty())
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-pause", Optional.empty())
.orElseThrow();
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
executionQueue.emit(execution2);
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
assertThat(execution1.getState().isPaused()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
assertTrue(firstExecutionLatch.await(10, TimeUnit.SECONDS));
assertTrue(secondExecutionLatch.await(10, TimeUnit.SECONDS));
receive.blockLast();
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(secondExecutionResult.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(secondExecutionResult.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
}
public void flowConcurrencyCancelPause() throws TimeoutException, QueueException, InterruptedException {
AtomicReference<String> firstExecutionId = new AtomicReference<>();
var firstExecutionResult = new AtomicReference<Execution>();
var secondExecutionResult = new AtomicReference<Execution>();
CountDownLatch firstExecLatch = new CountDownLatch(1);
CountDownLatch secondExecLatch = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (!"flow-concurrency-cancel-pause".equals(e.getLeft().getFlowId())){
return;
}
String currentId = e.getLeft().getId();
Type currentState = e.getLeft().getState().getCurrent();
if (firstExecutionId.get() == null) {
firstExecutionId.set(currentId);
}
if (currentId.equals(firstExecutionId.get())) {
if (currentState == State.Type.SUCCESS) {
firstExecutionResult.set(e.getLeft());
firstExecLatch.countDown();
}
} else {
if (currentState == State.Type.CANCELLED) {
secondExecutionResult.set(e.getLeft());
secondExecLatch.countDown();
}
}
});
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause");
public void flowConcurrencyCancelPause() throws QueueException {
Execution execution1 = runnerUtils.runOneUntilPaused(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause");
Flow flow = flowRepository
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-cancel-pause", Optional.empty())
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-cancel-pause", Optional.empty())
.orElseThrow();
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
executionQueue.emit(execution2);
Execution secondExecutionResult = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.CANCELLED), execution2);
Execution firstExecutionResult = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
assertThat(execution1.getState().isPaused()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
assertTrue(firstExecLatch.await(10, TimeUnit.SECONDS));
assertTrue(secondExecLatch.await(10, TimeUnit.SECONDS));
receive.blockLast();
assertThat(firstExecutionResult.get().getId()).isEqualTo(execution1.getId());
assertThat(firstExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.get().getId()).isEqualTo(execution2.getId());
assertThat(secondExecutionResult.get().getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
assertThat(secondExecutionResult.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(secondExecutionResult.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
assertThat(firstExecutionResult.getId()).isEqualTo(execution1.getId());
assertThat(firstExecutionResult.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(secondExecutionResult.getId()).isEqualTo(execution2.getId());
assertThat(secondExecutionResult.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
assertThat(secondExecutionResult.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(secondExecutionResult.getState().getHistories().get(1).getState()).isEqualTo(State.Type.CANCELLED);
}
public void flowConcurrencyWithForEachItem() throws TimeoutException, QueueException, InterruptedException, URISyntaxException, IOException {
URI file = storageUpload();
public void flowConcurrencyWithForEachItem(String tenantId) throws QueueException, URISyntaxException, IOException {
URI file = storageUpload(tenantId);
Map<String, Object> inputs = Map.of("file", file.toString(), "batch", 4);
Execution forEachItem = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-for-each-item", null,
Execution forEachItem = runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-for-each-item", null,
(flow, execution1) -> flowIO.readExecutionInputs(flow, execution1, inputs), Duration.ofSeconds(5));
assertThat(forEachItem.getState().getCurrent()).isEqualTo(Type.RUNNING);
Set<String> executionIds = new HashSet<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if ("flow-concurrency-queue".equals(e.getLeft().getFlowId()) && e.getLeft().getState().isRunning()) {
executionIds.add(e.getLeft().getId());
}
});
// wait a little to be sure there are not too many executions started
Thread.sleep(500);
assertThat(executionIds).hasSize(1);
receive.blockLast();
Execution terminated = runnerUtils.awaitExecution(e -> e.getId().equals(forEachItem.getId()) && e.getState().isTerminated(), () -> {}, Duration.ofSeconds(10));
Execution terminated = runnerUtils.awaitExecution(e -> e.getState().isTerminated(),forEachItem);
assertThat(terminated.getState().getCurrent()).isEqualTo(Type.SUCCESS);
List<Execution> executions = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-queue");
assertThat(executions).extracting(e -> e.getState().getCurrent()).containsOnly(Type.SUCCESS);
assertThat(executions.stream()
.map(e -> e.getState().getHistories())
.flatMap(List::stream)
.map(History::getState)
.toList()).contains(Type.QUEUED);
}
public void flowConcurrencyQueueRestarted() throws Exception {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE,
"flow-concurrency-queue-fail", null, null, Duration.ofSeconds(30));
Flow flow = flowRepository
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-fail", Optional.empty())
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-fail", Optional.empty())
.orElseThrow();
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
executionQueue.emit(execution2);
runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.RUNNING), execution2);
assertThat(execution1.getState().isRunning()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
var executionResult1 = new AtomicReference<Execution>();
var executionResult2 = new AtomicReference<Execution>();
CountDownLatch latch1 = new CountDownLatch(2);
AtomicReference<Execution> failedExecution = new AtomicReference<>();
CountDownLatch latch2 = new CountDownLatch(1);
CountDownLatch latch3 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution1.getId())) {
executionResult1.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
failedExecution.set(e.getLeft());
latch1.countDown();
}
}
if (e.getLeft().getId().equals(execution2.getId())) {
executionResult2.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
latch2.countDown();
}
if (e.getLeft().getState().getCurrent() == Type.FAILED) {
latch3.countDown();
}
}
});
assertTrue(latch2.await(1, TimeUnit.MINUTES));
assertThat(failedExecution.get()).isNotNull();
// here the first fail and the second is now running.
// we restart the first one, it should be queued then fail again.
Execution restarted = executionService.restart(failedExecution.get(), null);
executionQueue.emit(restarted);
Execution failedExecution = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution1);
Execution restarted = executionService.restart(failedExecution, null);
Execution executionResult1 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), restarted);
Execution executionResult2 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.FAILED), execution2);
assertTrue(latch3.await(1, TimeUnit.MINUTES));
assertTrue(latch1.await(1, TimeUnit.MINUTES));
receive.blockLast();
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(Type.FAILED);
assertThat(executionResult1.getState().getCurrent()).isEqualTo(Type.FAILED);
// it should have been queued after restarted
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
assertThat(executionResult1.get().getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(Type.FAILED);
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.RESTARTED)).isTrue();
assertThat(executionResult1.getState().getHistories().stream().anyMatch(history -> history.getState() == Type.QUEUED)).isTrue();
assertThat(executionResult2.getState().getCurrent()).isEqualTo(Type.FAILED);
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
}
public void flowConcurrencyQueueAfterExecution() throws TimeoutException, QueueException, InterruptedException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
public void flowConcurrencyQueueAfterExecution() throws QueueException {
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", null, null, Duration.ofSeconds(30));
Flow flow = flowRepository
.findById(MAIN_TENANT, "io.kestra.tests", "flow-concurrency-queue-after-execution", Optional.empty())
.findById(MAIN_TENANT, NAMESPACE, "flow-concurrency-queue-after-execution", Optional.empty())
.orElseThrow();
Execution execution2 = Execution.newExecution(flow, null, null, Optional.empty());
executionQueue.emit(execution2);
Execution executionResult2 = runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution2);
Execution executionResult1 = runnerUtils.awaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution1);
assertThat(execution1.getState().isRunning()).isTrue();
assertThat(execution2.getState().getCurrent()).isEqualTo(State.Type.CREATED);
var executionResult1 = new AtomicReference<Execution>();
var executionResult2 = new AtomicReference<Execution>();
CountDownLatch latch1 = new CountDownLatch(1);
CountDownLatch latch2 = new CountDownLatch(1);
CountDownLatch latch3 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution1.getId())) {
executionResult1.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch1.countDown();
}
}
if (e.getLeft().getId().equals(execution2.getId())) {
executionResult2.set(e.getLeft());
if (e.getLeft().getState().getCurrent() == State.Type.RUNNING) {
latch2.countDown();
}
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch3.countDown();
}
}
});
assertTrue(latch1.await(1, TimeUnit.MINUTES));
assertTrue(latch2.await(1, TimeUnit.MINUTES));
assertTrue(latch3.await(1, TimeUnit.MINUTES));
receive.blockLast();
assertThat(executionResult1.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.get().getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.get().getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.get().getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
assertThat(executionResult1.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(executionResult2.getState().getHistories().getFirst().getState()).isEqualTo(State.Type.CREATED);
assertThat(executionResult2.getState().getHistories().get(1).getState()).isEqualTo(State.Type.QUEUED);
assertThat(executionResult2.getState().getHistories().get(2).getState()).isEqualTo(State.Type.RUNNING);
}
private URI storageUpload() throws URISyntaxException, IOException {
public void flowConcurrencySubflow(String tenantId) throws TimeoutException, QueueException {
runnerUtils.runOneUntilRunning(tenantId, NAMESPACE, "flow-concurrency-subflow", null, null, Duration.ofSeconds(30));
runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
List<Execution> subFlowExecs = runnerUtils.awaitFlowExecutionNumber(2, tenantId, NAMESPACE, "flow-concurrency-cancel");
assertThat(subFlowExecs).extracting(e -> e.getState().getCurrent()).containsExactlyInAnyOrder(Type.SUCCESS, Type.CANCELLED);
// run another execution to be sure that everything work (purge is correctly done)
Execution execution3 = runnerUtils.runOne(tenantId, NAMESPACE, "flow-concurrency-subflow");
assertThat(execution3.getState().getCurrent()).isEqualTo(Type.SUCCESS);
runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE, "flow-concurrency-cancel");
}
private URI storageUpload(String tenantId) throws URISyntaxException, IOException {
File tempFile = File.createTempFile("file", ".txt");
Files.write(tempFile.toPath(), content());
return storageInterface.put(
MAIN_TENANT,
tenantId,
null,
new URI("/file/storage/file.txt"),
new FileInputStream(tempFile)

View File

@@ -4,19 +4,22 @@ import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.flows.GenericFlow;
import io.kestra.core.models.property.Property;
import io.kestra.core.junit.annotations.KestraTest;
import lombok.SneakyThrows;
import io.kestra.core.utils.Await;
import io.kestra.core.utils.TestsUtils;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeoutException;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.services.FlowListenersInterface;
import io.kestra.plugin.core.debug.Return;
import io.kestra.core.utils.IdUtils;
import java.util.Collections;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import jakarta.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
@@ -24,11 +27,11 @@ abstract public class FlowListenersTest {
@Inject
protected FlowRepositoryInterface flowRepository;
protected static FlowWithSource create(String flowId, String taskId) {
protected static FlowWithSource create(String tenantId, String flowId, String taskId) {
FlowWithSource flow = FlowWithSource.builder()
.id(flowId)
.namespace("io.kestra.unittest")
.tenantId(MAIN_TENANT)
.tenantId(tenantId)
.revision(1)
.tasks(Collections.singletonList(Return.builder()
.id(taskId)
@@ -39,88 +42,65 @@ abstract public class FlowListenersTest {
return flow.toBuilder().source(flow.sourceOrGenerateIfNull()).build();
}
public void suite(FlowListenersInterface flowListenersService) {
private static final Logger LOG = LoggerFactory.getLogger(FlowListenersTest.class);
public void suite(FlowListenersInterface flowListenersService) throws TimeoutException {
String tenant = TestsUtils.randomTenant(this.getClass().getSimpleName());
flowListenersService.run();
AtomicInteger count = new AtomicInteger();
var ref = new Ref();
flowListenersService.listen(flows -> {
count.set(flows.size());
ref.countDownLatch.countDown();
});
flowListenersService.listen(flows -> count.set(getFlowsForTenant(flowListenersService, tenant).size()));
// initial state
wait(ref, () -> {
assertThat(count.get()).isZero();
assertThat(flowListenersService.flows().size()).isZero();
});
LOG.info("-----------> wait for zero");
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
// resend on startup done for kafka
LOG.info("-----------> wait for zero kafka");
if (flowListenersService.getClass().getName().equals("io.kestra.ee.runner.kafka.KafkaFlowListeners")) {
wait(ref, () -> {
assertThat(count.get()).isZero();
assertThat(flowListenersService.flows().size()).isZero();
});
Await.until(() -> count.get() == 0, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isZero();
}
// create first
FlowWithSource first = create("first_" + IdUtils.create(), "test");
FlowWithSource firstUpdated = create(first.getId(), "test2");
LOG.info("-----------> create fist flow");
FlowWithSource first = create(tenant, "first_" + IdUtils.create(), "test");
FlowWithSource firstUpdated = create(tenant, first.getId(), "test2");
flowRepository.create(GenericFlow.of(first));
wait(ref, () -> {
assertThat(count.get()).isEqualTo(1);
assertThat(flowListenersService.flows().size()).isEqualTo(1);
});
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
// create the same id than first, no additional flows
first = flowRepository.update(GenericFlow.of(firstUpdated), first);
wait(ref, () -> {
assertThat(count.get()).isEqualTo(1);
assertThat(flowListenersService.flows().size()).isEqualTo(1);
//assertThat(flowListenersService.flows().getFirst().getFirst().getId(), is("test2"));
});
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
FlowWithSource second = create("second_" + IdUtils.create(), "test");
FlowWithSource second = create(tenant, "second_" + IdUtils.create(), "test");
// create a new one
flowRepository.create(GenericFlow.of(second));
wait(ref, () -> {
assertThat(count.get()).isEqualTo(2);
assertThat(flowListenersService.flows().size()).isEqualTo(2);
});
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
// delete first
FlowWithSource deleted = flowRepository.delete(first);
wait(ref, () -> {
assertThat(count.get()).isEqualTo(1);
assertThat(flowListenersService.flows().size()).isEqualTo(1);
});
Await.until(() -> count.get() == 1, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(1);
// restore must works
flowRepository.create(GenericFlow.of(first));
wait(ref, () -> {
assertThat(count.get()).isEqualTo(2);
assertThat(flowListenersService.flows().size()).isEqualTo(2);
});
Await.until(() -> count.get() == 2, Duration.ofMillis(10), Duration.ofSeconds(5));
assertThat(getFlowsForTenant(flowListenersService, tenant).size()).isEqualTo(2);
FlowWithSource withTenant = first.toBuilder().tenantId("some-tenant").build();
flowRepository.create(GenericFlow.of(withTenant));
wait(ref, () -> {
assertThat(count.get()).isEqualTo(3);
assertThat(flowListenersService.flows().size()).isEqualTo(3);
});
}
public static class Ref {
CountDownLatch countDownLatch = new CountDownLatch(1);
public List<FlowWithSource> getFlowsForTenant(FlowListenersInterface flowListenersService, String tenantId){
return flowListenersService.flows().stream()
.filter(f -> tenantId.equals(f.getTenantId()))
.toList();
}
@SneakyThrows
private void wait(Ref ref, Runnable run) {
ref.countDownLatch.await(60, TimeUnit.SECONDS);
run.run();
ref.countDownLatch = new CountDownLatch(1);
}
}

View File

@@ -2,82 +2,61 @@ package io.kestra.core.runners;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.queues.QueueException;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import reactor.core.publisher.Flux;
import java.util.ArrayList;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Singleton
public class FlowTriggerCaseTest {
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface<Execution> executionQueue;
public static final String NAMESPACE = "io.kestra.tests.trigger";
@Inject
protected RunnerUtils runnerUtils;
protected TestRunnerUtils runnerUtils;
public void trigger() throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(3);
AtomicReference<Execution> flowListener = new AtomicReference<>();
AtomicReference<Execution> flowListenerNoInput = new AtomicReference<>();
AtomicReference<Execution> flowListenerNamespace = new AtomicReference<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS) {
if (flowListenerNoInput.get() == null && execution.getFlowId().equals("trigger-flow-listener-no-inputs")) {
flowListenerNoInput.set(execution);
countDownLatch.countDown();
} else if (flowListener.get() == null && execution.getFlowId().equals("trigger-flow-listener")) {
flowListener.set(execution);
countDownLatch.countDown();
} else if (flowListenerNamespace.get() == null && execution.getFlowId().equals("trigger-flow-listener-namespace-condition")) {
flowListenerNamespace.set(execution);
countDownLatch.countDown();
}
}
});
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger", "trigger-flow");
public void trigger(String tenantId) throws InterruptedException, TimeoutException, QueueException {
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE, "trigger-flow");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
receive.blockLast();
Execution flowListenerNoInput = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
"trigger-flow-listener-no-inputs");
Execution flowListener = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
"trigger-flow-listener");
Execution flowListenerNamespace = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS), tenantId, NAMESPACE,
"trigger-flow-listener-namespace-condition");
assertThat(flowListener.get().getTaskRunList().size()).isEqualTo(1);
assertThat(flowListener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(flowListener.get().getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
assertThat(flowListener.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
assertThat(flowListener.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
assertThat(flowListener.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
assertThat(flowListenerNoInput.get().getTaskRunList().size()).isEqualTo(1);
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
assertThat(flowListenerNoInput.get().getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
assertThat(flowListenerNoInput.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(flowListener.getTaskRunList().size()).isEqualTo(1);
assertThat(flowListener.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(flowListener.getTaskRunList().getFirst().getOutputs().get("value")).isEqualTo("childs: from parents: " + execution.getId());
assertThat(flowListener.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
assertThat(flowListener.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
assertThat(flowListener.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
assertThat(flowListenerNamespace.get().getTaskRunList().size()).isEqualTo(1);
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
assertThat(flowListenerNoInput.getTaskRunList().size()).isEqualTo(1);
assertThat(flowListenerNoInput.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
assertThat(flowListenerNoInput.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
assertThat(flowListenerNoInput.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-flow");
assertThat(flowListenerNoInput.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(flowListenerNamespace.getTaskRunList().size()).isEqualTo(1);
assertThat(flowListenerNamespace.getTrigger().getVariables().get("namespace")).isEqualTo(NAMESPACE);
// it will be triggered for 'trigger-flow' or any of the 'trigger-flow-listener*', so we only assert that it's one of them
assertThat(flowListenerNamespace.get().getTrigger().getVariables().get("flowId"))
assertThat(flowListenerNamespace.getTrigger().getVariables().get("flowId"))
.satisfiesAnyOf(
arg -> assertThat(arg).isEqualTo("trigger-flow"),
arg -> assertThat(arg).isEqualTo("trigger-flow-listener-no-inputs"),
@@ -85,56 +64,43 @@ public class FlowTriggerCaseTest {
);
}
public void triggerWithPause() throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(4);
List<Execution> flowListeners = new ArrayList<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-pause")) {
flowListeners.add(execution);
countDownLatch.countDown();
}
});
public void triggerWithPause() throws TimeoutException, QueueException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.pause", "trigger-flow-with-pause");
assertThat(execution.getTaskRunList().size()).isEqualTo(3);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
receive.blockLast();
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
4,
MAIN_TENANT,
"io.kestra.tests.trigger.pause",
"trigger-flow-listener-with-pause");
assertThat(flowListeners.size()).isEqualTo(4);
assertThat(flowListeners.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
assertThat(flowListeners.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
assertThat(flowListeners.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
assertThat(flowListeners.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
assertThat(triggeredExec.size()).isEqualTo(4);
List<Execution> sortedExecs = triggeredExec.stream()
.sorted(Comparator.comparing(e -> e.getState().getEndDate().orElse(Instant.now())))
.toList();
assertThat(sortedExecs.get(0).getOutputs().get("status")).isEqualTo("RUNNING");
assertThat(sortedExecs.get(1).getOutputs().get("status")).isEqualTo("PAUSED");
assertThat(sortedExecs.get(2).getOutputs().get("status")).isEqualTo("RUNNING");
assertThat(sortedExecs.get(3).getOutputs().get("status")).isEqualTo("SUCCESS");
}
public void triggerWithConcurrencyLimit() throws QueueException, TimeoutException, InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch(5);
List<Execution> flowListeners = new ArrayList<>();
public void triggerWithConcurrencyLimit(String tenantId) throws QueueException, TimeoutException {
Execution execution1 = runnerUtils.runOneUntilRunning(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
Execution execution2 = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId().equals("trigger-flow-listener-with-concurrency-limit")) {
flowListeners.add(execution);
countDownLatch.countDown();
}
});
List<Execution> triggeredExec = runnerUtils.awaitFlowExecutionNumber(
5,
tenantId,
"io.kestra.tests.trigger.concurrency",
"trigger-flow-listener-with-concurrency-limit");
Execution execution1 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
Execution execution2 = runnerUtils.runOneUntilRunning(MAIN_TENANT, "io.kestra.tests.trigger.concurrency", "trigger-flow-with-concurrency-limit");
assertTrue(countDownLatch.await(15, TimeUnit.SECONDS));
receive.blockLast();
assertThat(flowListeners.size()).isEqualTo(5);
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
assertThat(flowListeners.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
assertThat(triggeredExec.size()).isEqualTo(5);
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution1.getId()))).isTrue();
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("QUEUED") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("RUNNING") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
assertThat(triggeredExec.stream().anyMatch(e -> e.getOutputs().get("status").equals("SUCCESS") && e.getOutputs().get("executionId").equals(execution2.getId()))).isTrue();
}
}

View File

@@ -17,6 +17,9 @@ import io.kestra.core.storages.StorageInterface;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Test;
import jakarta.validation.ConstraintViolationException;
@@ -36,6 +39,7 @@ import java.util.concurrent.TimeoutException;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@KestraTest(startRunner = true)
public class InputsTest {
@@ -44,7 +48,7 @@ public class InputsTest {
private QueueInterface<LogEntry> logQueue;
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
public static Map<String, Object> inputs = ImmutableMap.<String, Object>builder()
.put("string", "myString")
@@ -90,8 +94,8 @@ public class InputsTest {
@Inject
private FlowInputOutput flowInputOutput;
private Map<String, Object> typedInputs(Map<String, Object> map) {
return typedInputs(map, flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "inputs").get());
private Map<String, Object> typedInputs(Map<String, Object> map, String tenantId) {
return typedInputs(map, flowRepository.findById(tenantId, "io.kestra.tests", "inputs").get());
}
private Map<String, Object> typedInputs(Map<String, Object> map, Flow flow) {
@@ -100,7 +104,7 @@ public class InputsTest {
Execution.builder()
.id("test")
.namespace(flow.getNamespace())
.tenantId(MAIN_TENANT)
.tenantId(flow.getTenantId())
.flowRevision(1)
.flowId(flow.getId())
.build(),
@@ -113,25 +117,25 @@ public class InputsTest {
void missingRequired() {
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
inputs.put("string", null);
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(inputs, MAIN_TENANT));
assertThat(e.getMessage()).contains("Invalid input for `string`, missing required input, but received `null`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant")
void nonRequiredNoDefaultNoValueIsNull() {
HashMap<String, Object> inputsWithMissingOptionalInput = new HashMap<>(inputs);
inputsWithMissingOptionalInput.remove("bool");
assertThat(typedInputs(inputsWithMissingOptionalInput).containsKey("bool")).isTrue();
assertThat(typedInputs(inputsWithMissingOptionalInput).get("bool")).isNull();
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").containsKey("bool")).isTrue();
assertThat(typedInputs(inputsWithMissingOptionalInput, "tenant").get("bool")).isNull();
}
@SuppressWarnings("unchecked")
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant1")
void allValidInputs() throws URISyntaxException, IOException {
Map<String, Object> typeds = typedInputs(inputs);
Map<String, Object> typeds = typedInputs(inputs, "tenant1");
assertThat(typeds.get("string")).isEqualTo("myString");
assertThat(typeds.get("int")).isEqualTo(42);
@@ -143,7 +147,7 @@ public class InputsTest {
assertThat(typeds.get("time")).isEqualTo(LocalTime.parse("18:27:49"));
assertThat(typeds.get("duration")).isEqualTo(Duration.parse("PT5M6S"));
assertThat((URI) typeds.get("file")).isEqualTo(new URI("kestra:///io/kestra/tests/inputs/executions/test/inputs/file/application-test.yml"));
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get(MAIN_TENANT, null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
assertThat(CharStreams.toString(new InputStreamReader(storageInterface.get("tenant1", null, (URI) typeds.get("file"))))).isEqualTo(CharStreams.toString(new InputStreamReader(new FileInputStream((String) inputs.get("file")))));
assertThat(typeds.get("json")).isEqualTo(Map.of("a", "b"));
assertThat(typeds.get("uri")).isEqualTo("https://www.google.com");
assertThat(((Map<String, Object>) typeds.get("nested")).get("string")).isEqualTo("a string");
@@ -166,9 +170,9 @@ public class InputsTest {
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant2")
void allValidTypedInputs() {
Map<String, Object> typeds = typedInputs(inputs);
Map<String, Object> typeds = typedInputs(inputs, "tenant2");
typeds.put("int", 42);
typeds.put("float", 42.42F);
typeds.put("bool", false);
@@ -181,10 +185,10 @@ public class InputsTest {
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant3")
void inputFlow() throws TimeoutException, QueueException {
Execution execution = runnerUtils.runOne(
MAIN_TENANT,
"tenant3",
"io.kestra.tests",
"inputs",
null,
@@ -201,165 +205,165 @@ public class InputsTest {
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant4")
void inputValidatedStringBadValue() {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("validatedString", "foo");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant4"));
assertThat(e.getMessage()).contains("Invalid input for `validatedString`, it must match the pattern");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant5")
void inputValidatedIntegerBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedInt", "9");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant5"));
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be more than `10`, but received `9`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedInt", "21");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant5"));
assertThat(e.getMessage()).contains("Invalid input for `validatedInt`, it must be less than `20`, but received `21`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant6")
void inputValidatedDateBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDate", "2022-01-01");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant6"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be after `2023-01-01`, but received `2022-01-01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDate", "2024-01-01");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant6"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDate`, it must be before `2023-12-31`, but received `2024-01-01`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant7")
void inputValidatedDateTimeBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDateTime", "2022-01-01T00:00:00Z");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant7"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be after `2023-01-01T00:00:00Z`, but received `2022-01-01T00:00:00Z`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDateTime", "2024-01-01T00:00:00Z");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant7"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDateTime`, it must be before `2023-12-31T23:59:59Z`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant8")
void inputValidatedDurationBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedDuration", "PT1S");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant8"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be more than `PT10S`, but received `PT1S`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedDuration", "PT30S");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant8"));
assertThat(e.getMessage()).contains("Invalid input for `validatedDuration`, It must be less than `PT20S`, but received `PT30S`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant9")
void inputValidatedFloatBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedFloat", "0.01");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant9"));
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be more than `0.1`, but received `0.01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedFloat", "1.01");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant9"));
assertThat(e.getMessage()).contains("Invalid input for `validatedFloat`, it must be less than `0.5`, but received `1.01`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant10")
void inputValidatedTimeBadValue() {
HashMap<String, Object> mapMin = new HashMap<>(inputs);
mapMin.put("validatedTime", "00:00:01");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMin, "tenant10"));
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be after `01:00`, but received `00:00:01`");
HashMap<String, Object> mapMax = new HashMap<>(inputs);
mapMax.put("validatedTime", "14:00:00");
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax));
e = assertThrows(ConstraintViolationException.class, () -> typedInputs(mapMax, "tenant10"));
assertThat(e.getMessage()).contains("Invalid input for `validatedTime`, it must be before `11:59:59`, but received `14:00:00`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant11")
void inputFailed() {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("uri", "http:/bla");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant11"));
assertThat(e.getMessage()).contains("Invalid input for `uri`, Expected `URI` but received `http:/bla`, but received `http:/bla`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant12")
void inputEnumFailed() {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("enum", "INVALID");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant12"));
assertThat(e.getMessage()).isEqualTo("enum: Invalid input for `enum`, it must match the values `[ENUM_VALUE, OTHER_ONE]`, but received `INVALID`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant13")
void inputArrayFailed() {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("array", "[\"s1\", \"s2\"]");
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map));
ConstraintViolationException e = assertThrows(ConstraintViolationException.class, () -> typedInputs(map, "tenant13"));
assertThat(e.getMessage()).contains("Invalid input for `array`, Unable to parse array element as `INT` on `s1`, but received `[\"s1\", \"s2\"]`");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant14")
void inputEmptyJson() {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("json", "{}");
Map<String, Object> typeds = typedInputs(map);
Map<String, Object> typeds = typedInputs(map, "tenant14");
assertThat(typeds.get("json")).isInstanceOf(Map.class);
assertThat(((Map<?, ?>) typeds.get("json")).size()).isZero();
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant15")
void inputEmptyJsonFlow() throws TimeoutException, QueueException {
HashMap<String, Object> map = new HashMap<>(inputs);
map.put("json", "{}");
Execution execution = runnerUtils.runOne(
MAIN_TENANT,
"tenant15",
"io.kestra.tests",
"inputs",
null,
@@ -375,12 +379,20 @@ public class InputsTest {
}
@Test
@LoadFlows({"flows/valids/input-log-secret.yaml"})
void shouldNotLogSecretInput() throws TimeoutException, QueueException {
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {});
@LoadFlows(value = {"flows/valids/input-log-secret.yaml"}, tenantId = "tenant16")
void shouldNotLogSecretInput() throws TimeoutException, QueueException, InterruptedException {
AtomicReference<LogEntry> logEntry = new AtomicReference<>();
CountDownLatch countDownLatch = new CountDownLatch(1);
Flux<LogEntry> receive = TestsUtils.receive(logQueue, l -> {
LogEntry left = l.getLeft();
if (left.getTenantId().equals("tenant16")){
logEntry.set(left);
countDownLatch.countDown();
}
});
Execution execution = runnerUtils.runOne(
MAIN_TENANT,
"tenant16",
"io.kestra.tests",
"input-log-secret",
null,
@@ -390,20 +402,21 @@ public class InputsTest {
assertThat(execution.getTaskRunList()).hasSize(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
var logEntry = receive.blockLast();
assertThat(logEntry).isNotNull();
assertThat(logEntry.getMessage()).isEqualTo("These are my secrets: ****** - ******");
receive.blockLast();
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
assertThat(logEntry.get()).isNotNull();
assertThat(logEntry.get().getMessage()).isEqualTo("These are my secrets: ****** - ******");
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant17")
void fileInputWithFileDefault() throws IOException, QueueException, TimeoutException {
HashMap<String, Object> newInputs = new HashMap<>(InputsTest.inputs);
URI file = createFile();
newInputs.put("file", file);
Execution execution = runnerUtils.runOne(
MAIN_TENANT,
"tenant17",
"io.kestra.tests",
"inputs",
null,
@@ -415,14 +428,14 @@ public class InputsTest {
}
@Test
@LoadFlows({"flows/valids/inputs.yaml"})
@LoadFlows(value = {"flows/valids/inputs.yaml"}, tenantId = "tenant18")
void fileInputWithNsfile() throws IOException, QueueException, TimeoutException {
HashMap<String, Object> inputs = new HashMap<>(InputsTest.inputs);
URI file = createNsFile(false);
URI file = createNsFile(false, "tenant18");
inputs.put("file", file);
Execution execution = runnerUtils.runOne(
MAIN_TENANT,
"tenant18",
"io.kestra.tests",
"inputs",
null,
@@ -439,11 +452,11 @@ public class InputsTest {
return tempFile.toPath().toUri();
}
private URI createNsFile(boolean nsInAuthority) throws IOException {
private URI createNsFile(boolean nsInAuthority, String tenantId) throws IOException {
String namespace = "io.kestra.tests";
String filePath = "file.txt";
storageInterface.createDirectory(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
storageInterface.put(MAIN_TENANT, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
storageInterface.createDirectory(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace)));
storageInterface.put(tenantId, namespace, URI.create(StorageContext.namespaceFilePrefix(namespace) + "/" + filePath), new ByteArrayInputStream("Hello World".getBytes()));
return URI.create("nsfile://" + (nsInAuthority ? namespace : "") + "/" + filePath);
}
}

View File

@@ -14,15 +14,17 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Objects;
import java.util.concurrent.TimeoutException;
import org.junit.jupiter.api.parallel.ExecutionMode;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
@KestraTest(startRunner = true)
class ListenersTest {
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
@Inject
private LocalFlowRepositoryLoader repositoryLoader;

View File

@@ -1,243 +1,168 @@
package io.kestra.core.runners;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.queues.QueueException;
import io.kestra.core.utils.TestsUtils;
import io.kestra.core.repositories.ArrayListTotal;
import io.kestra.core.repositories.ExecutionRepositoryInterface;
import io.micronaut.context.ApplicationContext;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.repositories.FlowRepositoryInterface;
import java.time.Duration;
import java.util.List;
import io.micronaut.data.model.Pageable;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import reactor.core.publisher.Flux;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Singleton
public class MultipleConditionTriggerCaseTest {
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface<Execution> executionQueue;
public static final String NAMESPACE = "io.kestra.tests.trigger";
@Inject
protected RunnerUtils runnerUtils;
protected TestRunnerUtils runnerUtils;
@Inject
protected FlowRepositoryInterface flowRepository;
@Inject
protected ExecutionRepositoryInterface executionRepository;
@Inject
protected ApplicationContext applicationContext;
public void trigger() throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(3);
ConcurrentHashMap<String, Execution> ended = new ConcurrentHashMap<>();
List<String> watchedExecutions = List.of("trigger-multiplecondition-flow-a",
"trigger-multiplecondition-flow-b",
"trigger-multiplecondition-listener"
);
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (watchedExecutions.contains(execution.getFlowId()) && execution.getState().getCurrent() == State.Type.SUCCESS) {
ended.put(execution.getId(), execution);
countDownLatch.countDown();
}
});
// first one
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
"trigger-multiplecondition-flow-a", Duration.ofSeconds(60));
Execution execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-a");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// wait a little to be sure that the trigger is not launching execution
Thread.sleep(1000);
assertThat(ended.size()).isEqualTo(1);
ArrayListTotal<Execution> flowBExecutions = executionRepository.findByFlowId(MAIN_TENANT,
NAMESPACE, "trigger-multiplecondition-flow-b", Pageable.UNPAGED);
ArrayListTotal<Execution> listenerExecutions = executionRepository.findByFlowId(MAIN_TENANT,
NAMESPACE, "trigger-multiplecondition-listener", Pageable.UNPAGED);
assertThat(flowBExecutions).isEmpty();
assertThat(listenerExecutions).isEmpty();
// second one
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
"trigger-multiplecondition-flow-b", Duration.ofSeconds(60));
execution = runnerUtils.runOne(MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-flow-b");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// trigger is done
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
receive.blockLast();
assertThat(ended.size()).isEqualTo(3);
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests.trigger",
"trigger-multiplecondition-listener").orElseThrow();
Execution triggerExecution = ended.entrySet()
.stream()
.filter(e -> e.getValue().getFlowId().equals(flow.getId()))
.findFirst()
.map(Map.Entry::getValue)
.orElseThrow();
Execution triggerExecution = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS),
MAIN_TENANT, NAMESPACE, "trigger-multiplecondition-listener");
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(triggerExecution.getTrigger().getVariables().get("executionId")).isEqualTo(execution.getId());
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo("io.kestra.tests.trigger");
assertThat(triggerExecution.getTrigger().getVariables().get("namespace")).isEqualTo(
NAMESPACE);
assertThat(triggerExecution.getTrigger().getVariables().get("flowId")).isEqualTo("trigger-multiplecondition-flow-b");
}
public void failed() throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(1);
AtomicReference<Execution> listener = new AtomicReference<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getFlowId().equals("trigger-flow-listener-namespace-condition")
&& execution.getState().getCurrent().isTerminated()) {
listener.set(execution);
countDownLatch.countDown();
}
});
public void failed(String tenantId) throws InterruptedException, TimeoutException, QueueException {
// first one
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
"trigger-multiplecondition-flow-c", Duration.ofSeconds(60));
Execution execution = runnerUtils.runOne(tenantId, NAMESPACE,
"trigger-multiplecondition-flow-c");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait a little to be sure that the trigger is not launching execution
Thread.sleep(1000);
assertThat(listener.get()).isNull();
ArrayListTotal<Execution> byFlowId = executionRepository.findByFlowId(tenantId, NAMESPACE,
"trigger-multiplecondition-flow-d", Pageable.UNPAGED);
assertThat(byFlowId).isEmpty();
// second one
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger",
"trigger-multiplecondition-flow-d", Duration.ofSeconds(60));
execution = runnerUtils.runOne(tenantId, NAMESPACE,
"trigger-multiplecondition-flow-d");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
Execution triggerExecution = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS),
tenantId, NAMESPACE, "trigger-flow-listener-namespace-condition");
// trigger was not done
assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
receive.blockLast();
assertThat(listener.get()).isNotNull();
assertThat(listener.get().getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
public void flowTriggerPreconditions()
throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(1);
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
.equals("flow-trigger-preconditions-flow-listen")) {
flowTrigger.set(execution);
countDownLatch.countDown();
}
});
public void flowTriggerPreconditions() throws TimeoutException, QueueException {
// flowA
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
"flow-trigger-preconditions-flow-a");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// flowB: we trigger it two times, as flow-trigger-flow-preconditions-flow-listen is configured with resetOnSuccess: false it should be triggered two times
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
"flow-trigger-preconditions-flow-a");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
"flow-trigger-preconditions-flow-b");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// trigger is done
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
receive.blockLast();
assertThat(flowTrigger.get()).isNotNull();
Execution triggerExecution = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS),
MAIN_TENANT, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
Execution triggerExecution = flowTrigger.get();
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
}
public void flowTriggerPreconditionsMergeOutputs() throws QueueException, TimeoutException, InterruptedException {
public void flowTriggerPreconditionsMergeOutputs(String tenantId) throws QueueException, TimeoutException {
// we do the same as in flowTriggerPreconditions() but we trigger flows in the opposite order to be sure that outputs are merged
CountDownLatch countDownLatch = new CountDownLatch(1);
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
.equals("flow-trigger-preconditions-flow-listen")) {
flowTrigger.set(execution);
countDownLatch.countDown();
}
});
// flowB
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-b", Duration.ofSeconds(60));
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-b");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// flowA
execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-a", Duration.ofSeconds(60));
execution = runnerUtils.runOne(tenantId, "io.kestra.tests.trigger.preconditions",
"flow-trigger-preconditions-flow-a");
assertThat(execution.getTaskRunList().size()).isEqualTo(1);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// trigger is done
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
receive.blockLast();
assertThat(flowTrigger.get()).isNotNull();
Execution triggerExecution = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS),
tenantId, "io.kestra.tests.trigger.preconditions", "flow-trigger-preconditions-flow-listen");
Execution triggerExecution = flowTrigger.get();
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(triggerExecution.getTrigger().getVariables().get("outputs")).isNotNull();
assertThat((Map<String, Object>) triggerExecution.getTrigger().getVariables().get("outputs")).containsEntry("some", "value");
}
public void flowTriggerOnPaused()
throws InterruptedException, TimeoutException, QueueException {
CountDownLatch countDownLatch = new CountDownLatch(1);
AtomicReference<Execution> flowTrigger = new AtomicReference<>();
Flux<Execution> receive = TestsUtils.receive(executionQueue, either -> {
Execution execution = either.getLeft();
if (execution.getState().getCurrent() == State.Type.SUCCESS && execution.getFlowId()
.equals("flow-trigger-paused-listen")) {
flowTrigger.set(execution);
countDownLatch.countDown();
}
});
public void flowTriggerOnPaused() throws TimeoutException, QueueException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests.trigger.paused",
"flow-trigger-paused-flow", Duration.ofSeconds(60));
"flow-trigger-paused-flow");
assertThat(execution.getTaskRunList().size()).isEqualTo(2);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// trigger is done
assertTrue(countDownLatch.await(1, TimeUnit.SECONDS));
receive.blockLast();
assertThat(flowTrigger.get()).isNotNull();
Execution triggerExecution = runnerUtils.awaitFlowExecution(
e -> e.getState().getCurrent().equals(Type.SUCCESS),
MAIN_TENANT, "io.kestra.tests.trigger.paused", "flow-trigger-paused-listen");
Execution triggerExecution = flowTrigger.get();
assertThat(triggerExecution.getTaskRunList().size()).isEqualTo(1);
assertThat(triggerExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}

View File

@@ -30,7 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat;
@Singleton
public class PluginDefaultsCaseTest {
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
public void taskDefaults() throws TimeoutException, QueueException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "plugin-defaults", Duration.ofSeconds(60));

View File

@@ -4,29 +4,19 @@ import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.queues.QueueException;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.services.ExecutionService;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import reactor.core.publisher.Flux;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static io.kestra.core.utils.Rethrow.throwRunnable;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Singleton
@@ -35,38 +25,30 @@ public class RestartCaseTest {
private FlowRepositoryInterface flowRepository;
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
@Inject
private ExecutionService executionService;
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
private QueueInterface<Execution> executionQueue;
public void restartFailedThenSuccess() throws Exception {
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "restart_last_failed").orElseThrow();
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId(), Duration.ofSeconds(60));
Execution firstExecution = runnerUtils.runOne(MAIN_TENANT, flow.getNamespace(), flow.getId());
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
assertThat(firstExecution.getTaskRunList()).hasSize(3);
assertThat(firstExecution.getTaskRunList().get(2).getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(3);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
execution -> execution.getState().getCurrent() == State.Type.SUCCESS && execution.getId().equals(firstExecution.getId()),
throwRunnable(() -> {
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(3);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
executionQueue.emit(restartedExec);
}),
Duration.ofSeconds(60)
restartedExec
);
assertThat(finishedRestartedExecution).isNotNull();
@@ -93,19 +75,16 @@ public class RestartCaseTest {
assertThat(firstExecution.getTaskRunList().getFirst().getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.getTaskRunList().getFirst().getAttempts().size() == 2,
throwRunnable(() -> {
Execution restartedExec = executionService.restart(firstExecution, null);
executionQueue.emit(restartedExec);
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(1);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
}),
Duration.ofSeconds(60)
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(1);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.getTaskRunList().getFirst().getAttempts().size() == 2,
restartedExec
);
assertThat(finishedRestartedExecution).isNotNull();
@@ -128,19 +107,16 @@ public class RestartCaseTest {
assertThat(firstExecution.getTaskRunList().get(3).getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.findTaskRunsByTaskId("failStep").stream().findFirst().get().getAttempts().size() == 2,
throwRunnable(() -> {
Execution restartedExec = executionService.restart(firstExecution, null);
executionQueue.emit(restartedExec);
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(4);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
}),
Duration.ofSeconds(60)
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(4);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
execution -> execution.getState().getCurrent() == State.Type.FAILED && execution.findTaskRunsByTaskId("failStep").stream().findFirst().get().getAttempts().size() == 2,
restartedExec
);
assertThat(finishedRestartedExecution).isNotNull();
@@ -163,21 +139,19 @@ public class RestartCaseTest {
assertThat(firstExecution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
// wait
Execution restartedExec = executionService.replay(firstExecution, firstExecution.findTaskRunByTaskIdAndValue("2_end", List.of()).getId(), null);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
assertThat(restartedExec.getState().getHistories()).hasSize(4);
assertThat(restartedExec.getTaskRunList()).hasSize(20);
assertThat(restartedExec.getTaskRunList().get(19).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
assertThat(restartedExec.getId()).isNotEqualTo(firstExecution.getId());
assertThat(restartedExec.getTaskRunList().get(1).getId()).isNotEqualTo(firstExecution.getTaskRunList().get(1).getId());
Execution finishedRestartedExecution = runnerUtils.awaitChildExecution(
flow,
firstExecution,
throwRunnable(() -> {
Execution restartedExec = executionService.replay(firstExecution, firstExecution.findTaskRunByTaskIdAndValue("2_end", List.of()).getId(), null);
executionQueue.emit(restartedExec);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
assertThat(restartedExec.getState().getHistories()).hasSize(4);
assertThat(restartedExec.getTaskRunList()).hasSize(20);
assertThat(restartedExec.getTaskRunList().get(19).getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
assertThat(restartedExec.getId()).isNotEqualTo(firstExecution.getId());
assertThat(restartedExec.getTaskRunList().get(1).getId()).isNotEqualTo(firstExecution.getTaskRunList().get(1).getId());
}),
restartedExec,
Duration.ofSeconds(60)
);
@@ -195,71 +169,58 @@ public class RestartCaseTest {
Execution restart = executionService.restart(execution, null);
assertThat(restart.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution restartEnded = runnerUtils.awaitExecution(
Execution restartEnded = runnerUtils.emitAndAwaitExecution(
e -> e.getState().getCurrent() == State.Type.FAILED,
throwRunnable(() -> executionQueue.emit(restart)),
Duration.ofSeconds(120)
restart,
Duration.ofSeconds(60)
);
assertThat(restartEnded.getState().getCurrent()).isEqualTo(State.Type.FAILED);
Execution newRestart = executionService.restart(restartEnded, null);
restartEnded = runnerUtils.awaitExecution(
restartEnded = runnerUtils.emitAndAwaitExecution(
e -> e.getState().getCurrent() == State.Type.FAILED,
throwRunnable(() -> executionQueue.emit(newRestart)),
Duration.ofSeconds(120)
newRestart,
Duration.ofSeconds(60)
);
assertThat(restartEnded.getState().getCurrent()).isEqualTo(State.Type.FAILED);
}
public void restartSubflow() throws Exception {
CountDownLatch countDownLatch = new CountDownLatch(1);
Flux<Execution> receiveSubflows = TestsUtils.receive(executionQueue, either -> {
Execution subflowExecution = either.getLeft();
if (subflowExecution.getFlowId().equals("restart-child") && subflowExecution.getState().getCurrent().isFailed()) {
countDownLatch.countDown();
}
});
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "restart-parent");
assertThat(execution.getTaskRunList()).hasSize(3);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
// here we must have 1 failed subflows
assertTrue(countDownLatch.await(1, TimeUnit.MINUTES));
receiveSubflows.blockLast();
runnerUtils.awaitFlowExecution(e -> e.getState().getCurrent().isFailed(), MAIN_TENANT, "io.kestra.tests", "restart-child");
// there is 3 values so we must restart it 3 times to end the 3 subflows
CountDownLatch successLatch = new CountDownLatch(3);
receiveSubflows = TestsUtils.receive(executionQueue, either -> {
Execution subflowExecution = either.getLeft();
if (subflowExecution.getFlowId().equals("restart-child") && subflowExecution.getState().getCurrent().isSuccess()) {
successLatch.countDown();
}
});
Execution restarted1 = executionService.restart(execution, null);
execution = runnerUtils.awaitExecution(
execution = runnerUtils.emitAndAwaitExecution(
e -> e.getState().getCurrent() == State.Type.FAILED && e.getFlowId().equals("restart-parent"),
throwRunnable(() -> executionQueue.emit(restarted1)),
restarted1,
Duration.ofSeconds(10)
);
Execution restarted2 = executionService.restart(execution, null);
execution = runnerUtils.awaitExecution(
execution = runnerUtils.emitAndAwaitExecution(
e -> e.getState().getCurrent() == State.Type.FAILED && e.getFlowId().equals("restart-parent"),
throwRunnable(() -> executionQueue.emit(restarted2)),
restarted2,
Duration.ofSeconds(10)
);
Execution restarted3 = executionService.restart(execution, null);
execution = runnerUtils.awaitExecution(
execution = runnerUtils.emitAndAwaitExecution(
e -> e.getState().getCurrent() == State.Type.SUCCESS && e.getFlowId().equals("restart-parent"),
throwRunnable(() -> executionQueue.emit(restarted3)),
restarted3,
Duration.ofSeconds(10)
);
assertThat(execution.getTaskRunList()).hasSize(6);
assertTrue(successLatch.await(1, TimeUnit.MINUTES));
receiveSubflows.blockLast();
List<Execution> childExecutions = runnerUtils.awaitFlowExecutionNumber(3, MAIN_TENANT, "io.kestra.tests", "restart-child");
List<Execution> successfulRestart = childExecutions.stream()
.filter(e -> e.getState().getCurrent().equals(Type.SUCCESS)).toList();
assertThat(successfulRestart).hasSize(3);
}
public void restartFailedWithFinally() throws Exception {
@@ -272,18 +233,15 @@ public class RestartCaseTest {
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess() && execution.getId().equals(firstExecution.getId()),
throwRunnable(() -> {
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
executionQueue.emit(restartedExec);
}),
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess(),
restartedExec,
Duration.ofSeconds(60)
);
@@ -309,21 +267,18 @@ public class RestartCaseTest {
assertThat(firstExecution.getTaskRunList().get(1).getState().getCurrent()).isEqualTo(State.Type.FAILED);
// wait
Execution finishedRestartedExecution = runnerUtils.awaitExecution(
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess() && execution.getId().equals(firstExecution.getId()),
throwRunnable(() -> {
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
Execution restartedExec = executionService.restart(firstExecution, null);
assertThat(restartedExec).isNotNull();
assertThat(restartedExec.getId()).isEqualTo(firstExecution.getId());
assertThat(restartedExec.getParentId()).isNull();
assertThat(restartedExec.getTaskRunList().size()).isEqualTo(2);
assertThat(restartedExec.getState().getCurrent()).isEqualTo(State.Type.RESTARTED);
executionQueue.emit(restartedExec);
}),
Execution finishedRestartedExecution = runnerUtils.emitAndAwaitExecution(
execution -> executionService.isTerminated(flow, execution) && execution.getState().isSuccess(),
restartedExec,
Duration.ofSeconds(60)
);
assertThat(finishedRestartedExecution).isNotNull();
assertThat(finishedRestartedExecution.getId()).isEqualTo(firstExecution.getId());
assertThat(finishedRestartedExecution.getParentId()).isNull();

View File

@@ -11,6 +11,7 @@ import jakarta.inject.Inject;
import jakarta.inject.Named;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.parallel.ExecutionMode;
import org.slf4j.Logger;
import org.slf4j.event.Level;
import reactor.core.publisher.Flux;
@@ -25,6 +26,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import static org.assertj.core.api.Assertions.assertThat;
@KestraTest
@org.junit.jupiter.api.parallel.Execution(ExecutionMode.SAME_THREAD)
class RunContextLoggerTest {
@Inject
@Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)

View File

@@ -98,7 +98,7 @@ class RunContextTest {
private FlowInputOutput flowIO;
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
@Inject
protected LocalFlowRepositoryLoader repositoryLoader;

View File

@@ -1,13 +1,24 @@
package io.kestra.core.runners;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.DependsOn;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.Type;
import io.kestra.core.models.flows.input.BoolInput;
import io.kestra.core.models.property.Property;
import io.kestra.core.models.property.PropertyContext;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.triggers.AbstractTrigger;
import io.kestra.core.runners.pebble.functions.SecretFunction;
import io.kestra.core.utils.IdUtils;
import io.micronaut.context.ApplicationContext;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@@ -112,4 +123,25 @@ class RunVariablesTest {
assertThat(kestra.get("environment")).isEqualTo("test");
assertThat(kestra.get("url")).isEqualTo("http://localhost:8080");
}
}
@Test
void nonResolvableDynamicInputsShouldBeSkipped() throws IllegalVariableEvaluationException {
Map<String, Object> variables = new RunVariables.DefaultBuilder()
.withFlow(Flow
.builder()
.namespace("a.b")
.id("c")
.inputs(List.of(
BoolInput.builder().id("a").type(Type.BOOL).defaults(Property.ofValue(true)).build(),
BoolInput.builder().id("b").type(Type.BOOL).dependsOn(new DependsOn(List.of("a"), null)).defaults(Property.ofExpression("{{inputs.a == true}}")).build()
))
.build()
)
.withExecution(Execution.builder().id(IdUtils.create()).build())
.build(new RunContextLogger(), PropertyContext.create(new VariableRenderer(Mockito.mock(ApplicationContext.class), Mockito.mock(VariableRenderer.VariableConfiguration.class), Collections.emptyList())));
Assertions.assertEquals(Map.of(
"a", true
), variables.get("inputs"));
}
}

View File

@@ -16,7 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat;
@Singleton
public class SLATestCase {
@Inject
private RunnerUtils runnerUtils;
private TestRunnerUtils runnerUtils;
public void maxDurationSLAShouldFail() throws QueueException, TimeoutException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-max-duration-fail");
@@ -36,14 +36,14 @@ public class SLATestCase {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
public void executionConditionSLAShouldCancel() throws QueueException, TimeoutException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "CANCEL"));
public void executionConditionSLAShouldCancel(String tenantId) throws QueueException, TimeoutException {
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "CANCEL"));
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CANCELLED);
}
public void executionConditionSLAShouldLabel() throws QueueException, TimeoutException {
Execution execution = runnerUtils.runOne(MAIN_TENANT, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "LABEL"));
public void executionConditionSLAShouldLabel(String tenantId) throws QueueException, TimeoutException {
Execution execution = runnerUtils.runOne(tenantId, "io.kestra.tests", "sla-execution-condition", null, (f, e) -> Map.of("string", "LABEL"));
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
assertThat(execution.getLabels()).contains(new Label("sla", "violated"));

View File

@@ -3,54 +3,31 @@ package io.kestra.core.runners;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.State;
import io.kestra.core.models.flows.State.Type;
import io.kestra.core.queues.QueueException;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.repositories.FlowRepositoryInterface;
import io.kestra.core.utils.TestsUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
import reactor.core.publisher.Flux;
import java.time.ZonedDateTime;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Singleton
public class ScheduleDateCaseTest {
@Inject
private FlowRepositoryInterface flowRepository;
@Inject
@Named(QueueFactoryInterface.EXECUTION_NAMED)
protected QueueInterface<Execution> executionQueue;
private TestRunnerUtils runnerUtils;
public void shouldScheduleOnDate() throws QueueException, InterruptedException {
public void shouldScheduleOnDate(String tenantId) throws QueueException {
ZonedDateTime scheduleOn = ZonedDateTime.now().plusSeconds(1);
Flow flow = flowRepository.findById(MAIN_TENANT, "io.kestra.tests", "minimal").orElseThrow();
Flow flow = flowRepository.findById(tenantId, "io.kestra.tests", "minimal").orElseThrow();
Execution execution = Execution.newExecution(flow, null, null, Optional.of(scheduleOn));
this.executionQueue.emit(execution);
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CREATED);
assertThat(execution.getScheduleDate()).isEqualTo(scheduleOn.toInstant());
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.CREATED);
CountDownLatch latch1 = new CountDownLatch(1);
Flux<Execution> receive = TestsUtils.receive(executionQueue, e -> {
if (e.getLeft().getId().equals(execution.getId())) {
if (e.getLeft().getState().getCurrent() == State.Type.SUCCESS) {
latch1.countDown();
}
}
});
assertTrue(latch1.await(1, TimeUnit.MINUTES));
receive.blockLast();
runnerUtils.emitAndAwaitExecution(e -> e.getState().getCurrent().equals(Type.SUCCESS), execution);
}
}

View File

@@ -32,7 +32,7 @@ public class SkipExecutionCaseTest {
protected QueueInterface<Execution> executionQueue;
@Inject
protected RunnerUtils runnerUtils;
protected TestRunnerUtils runnerUtils;
@Inject
private ExecutionRepositoryInterface executionRepository;

Some files were not shown because too many files have changed in this diff Show More