fix(queue): change index to remove a sort

As when the flowable is itself in a flowable that process tasks concurrently like the ForEach when using a concurrency limit, it can be done multiple time with different values.

This can only occurs if the expression is using `taskRun.value`.

Fixes https://github.com/kestra-io/kestra-ee/issues/6055

core/src/main/java/io/kestra/core/models/property/Property.java

@@ -93,7 +93,7 @@ public class Property<T> {
      * @return a new {@link Property} without a pre-rendered value
      */
     public Property<T> skipCache() {
-        return Property.ofExpression(expression);
+        return new Property<>(expression, true);
     }
 
     /**

core/src/main/java/io/kestra/core/utils/NamespaceFilesUtils.java

@@ -32,10 +32,12 @@ public class NamespaceFilesUtils {
     private ExecutorsUtils executorsUtils;
 
     private ExecutorService executorService;
+    private int maxThreads;
 
     @PostConstruct
     public void postConstruct() {
-        this.executorService = executorsUtils.maxCachedThreadPool(Math.max(Runtime.getRuntime().availableProcessors() * 4, 32), "namespace-file");
+        this.maxThreads = Math.max(Runtime.getRuntime().availableProcessors() * 4, 32);
+        this.executorService = executorsUtils.maxCachedThreadPool(maxThreads, "namespace-file");
     }
 
     public void loadNamespaceFiles(
@@ -63,7 +65,11 @@ public class NamespaceFilesUtils {
           matchedNamespaceFiles.addAll(files);
         }
 
+        // Use half of the available threads to avoid impacting concurrent tasks
+        int parallelism = maxThreads / 2;
         Flux.fromIterable(matchedNamespaceFiles)
+            .parallel(parallelism)
+            .runOn(Schedulers.fromExecutorService(executorService))
             .doOnNext(throwConsumer(nsFile -> {
                 InputStream content = runContext.storage().getFile(nsFile.uri());
                 Path path = folderPerNamespace ?
@@ -71,7 +77,7 @@ public class NamespaceFilesUtils {
                     Path.of(nsFile.path());
                 runContext.workingDir().putFile(path, content, fileExistComportment);
             }))
-            .publishOn(Schedulers.fromExecutorService(executorService))
+            .sequential()
             .blockLast();
 
         Duration duration = stopWatch.getDuration();

core/src/main/java/io/kestra/plugin/core/flow/LoopUntil.java

@@ -157,7 +157,7 @@ public class LoopUntil extends Task implements FlowableTask<LoopUntil.Output> {
 
     public Instant nextExecutionDate(RunContext runContext, Execution execution, TaskRun parentTaskRun) throws IllegalVariableEvaluationException {
         if (!this.reachedMaximums(runContext, execution, parentTaskRun, false)) {
-            String continueLoop = runContext.render(this.condition).as(String.class).orElse(null);
+            String continueLoop = runContext.render(this.condition).skipCache().as(String.class).orElse(null);
             if (!TruthUtils.isTruthy(continueLoop)) {
                 return Instant.now().plus(runContext.render(this.getCheckFrequency().getInterval()).as(Duration.class).orElseThrow());
             }

core/src/main/java/io/kestra/plugin/core/flow/Switch.java

@@ -123,7 +123,7 @@ public class Switch extends Task implements FlowableTask<Switch.Output> {
     }
 
     private String rendererValue(RunContext runContext) throws IllegalVariableEvaluationException {
-        return runContext.render(this.value).as(String.class).orElseThrow();
+        return runContext.render(this.value).skipCache().as(String.class).orElseThrow();
     }
 
     @Override

core/src/test/java/io/kestra/plugin/core/flow/SwitchTest.java

@@ -1,9 +1,11 @@
 package io.kestra.plugin.core.flow;
 
 import static io.kestra.core.tenant.TenantService.MAIN_TENANT;
+import static org.assertj.core.api.Assertions.as;
 import static org.assertj.core.api.Assertions.assertThat;
 
 import com.google.common.collect.ImmutableMap;
+import io.kestra.core.junit.annotations.ExecuteFlow;
 import io.kestra.core.junit.annotations.KestraTest;
 import io.kestra.core.junit.annotations.LoadFlows;
 import io.kestra.core.models.executions.Execution;
@@ -100,4 +102,14 @@ class SwitchTest {
 
         assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.FAILED);
     }
+
+    @Test
+    @ExecuteFlow("flows/valids/switch-in-concurrent-loop.yaml")
+    void switchInConcurrentLoop(Execution execution) {
+        assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
+        assertThat(execution.getTaskRunList()).hasSize(5);
+        // we check that OOMCRM_EB_DD_000 and OOMCRM_EB_DD_001 have been processed once
+        assertThat(execution.getTaskRunList().stream().filter(t -> t.getTaskId().equals("OOMCRM_EB_DD_000")).count()).isEqualTo(1);
+        assertThat(execution.getTaskRunList().stream().filter(t -> t.getTaskId().equals("OOMCRM_EB_DD_001")).count()).isEqualTo(1);
+    }
 }

core/src/test/resources/flows/valids/switch-in-concurrent-loop.yaml

@@ -0,0 +1,23 @@
+id: switch-in-concurrent-loop
+namespace: io.kestra.tests
+
+tasks:
+  - id: iterate_and_check_name
+    type: io.kestra.plugin.core.flow.ForEach
+    tasks:
+      - id: switch
+        type: io.kestra.plugin.core.flow.Switch
+        value: "{{ taskrun.value }}"
+        cases:
+          "Alice":
+            - id: OOMCRM_EB_DD_000
+              type: io.kestra.plugin.core.log.Log
+              message: Alice
+          "Bob":
+            - id: OOMCRM_EB_DD_001
+              type: io.kestra.plugin.core.log.Log
+              message: Bob
+
+    values: ["Alice", "Bob"]
+
+    concurrencyLimit: 0

core/src/test/resources/flows/valids/working-directory-namespace-files-with-namespaces.yaml

@@ -13,18 +13,19 @@ tasks:
             - io.test.second
             - io.test.third
           enabled: true
+          folderPerNamespace: true
           exclude:
             - /ignore/**
         tasks:
           - id: t1
             type: io.kestra.core.tasks.test.Read
-            path: "/test/a/b/c/1.txt"
+            path: "/io.test.third/test/a/b/c/1.txt"
           - id: t2
             type: io.kestra.core.tasks.test.Read
-            path: "/a/b/c/2.txt"
+            path: "/io.test.second/a/b/c/2.txt"
           - id: t3
             type: io.kestra.core.tasks.test.Read
-            path: "/a/b/3.txt"
+            path: "/io.test.first/a/b/3.txt"
           - id: t4
             type: io.kestra.core.tasks.test.Read
             path: "/ignore/4.txt"

jdbc-mysql/src/main/java/io/kestra/runner/mysql/MysqlQueue.java

@@ -36,7 +36,7 @@ public class MysqlQueue<T> extends JdbcQueue<T> {
                 AbstractJdbcRepository.field("offset")
             )
             // force using the dedicated index, or it made a scan of the PK index
-            .from(this.table.useIndex("ix_type__consumers"))
+            .from(this.table.useIndex("ix_type__offset"))
             .where(AbstractJdbcRepository.field("type").eq(queueType()))
             .and(DSL.or(List.of(
                 AbstractJdbcRepository.field("consumers").isNull(),

jdbc/src/main/java/io/kestra/jdbc/runner/JdbcExecutor.java

@@ -176,7 +176,7 @@ public class JdbcExecutor implements ExecutorInterface {
 
     @Inject
     private AbstractJdbcWorkerJobRunningRepository workerJobRunningRepository;
-    
+
     @Inject
     private SLAMonitorStorage slaMonitorStorage;
 
@@ -658,21 +658,16 @@ public class JdbcExecutor implements ExecutorInterface {
                                                     workerTaskResults.add(new WorkerTaskResult(taskRun));
                                                 }
                                             }
-                                            /// flowable attempt state transition to running
+                                            // flowable attempt state transition to running
                                             if (workerTask.getTask().isFlowable()) {
-                                                List<TaskRunAttempt> attempts = Optional.ofNullable(workerTask.getTaskRun().getAttempts())
-                                                    .map(ArrayList::new)
-                                                    .orElseGet(ArrayList::new);
-
-
-                                                attempts.add(
-                                                    TaskRunAttempt.builder()
-                                                        .state(new State().withState(State.Type.RUNNING))
-                                                        .build()
-                                                );
-
                                                 TaskRun updatedTaskRun = workerTask.getTaskRun()
-                                                    .withAttempts(attempts)
+                                                    .withAttempts(
+                                                        List.of(
+                                                            TaskRunAttempt.builder()
+                                                                .state(new State().withState(State.Type.RUNNING))
+                                                                .build()
+                                                        )
+                                                    )
                                                     .withState(State.Type.RUNNING);
 
                                                 workerTaskResults.add(new WorkerTaskResult(updatedTaskRun));

ui/package-lock.json

@@ -10,7 +10,7 @@
             "hasInstallScript": true,
             "dependencies": {
                 "@js-joda/core": "^5.6.5",
-                "@kestra-io/ui-libs": "^0.0.263",
+                "@kestra-io/ui-libs": "^0.0.264",
                 "@vue-flow/background": "^1.3.2",
                 "@vue-flow/controls": "^1.1.2",
                 "@vue-flow/core": "^1.47.0",
@@ -2829,9 +2829,9 @@
             "license": "BSD-3-Clause"
         },
         "node_modules/@kestra-io/ui-libs": {
-            "version": "0.0.263",
-            "resolved": "https://registry.npmjs.org/@kestra-io/ui-libs/-/ui-libs-0.0.263.tgz",
-            "integrity": "sha512-j1rWqcQAK2CudNBkcDPjUXyaGFeBzJ7QEhPKFAbleHSw0N3QFu/iy0rFZxJNIMWRi1mGZBh74D6vL0OqQJkT2Q==",
+            "version": "0.0.264",
+            "resolved": "https://registry.npmjs.org/@kestra-io/ui-libs/-/ui-libs-0.0.264.tgz",
+            "integrity": "sha512-yUZDNaE0wUPOuEq/FL/TQBRd1fTV2dyM8s+VcGRjNSM1uv1uZcsSHro56/heHQx17lo00FDcPT7BMKEifrVhBg==",
             "dependencies": {
                 "@nuxtjs/mdc": "^0.17.3",
                 "@popperjs/core": "^2.11.8",

ui/package.json

@@ -24,7 +24,7 @@
     },
     "dependencies": {
         "@js-joda/core": "^5.6.5",
-        "@kestra-io/ui-libs": "^0.0.263",
+        "@kestra-io/ui-libs": "^0.0.264",
         "@vue-flow/background": "^1.3.2",
         "@vue-flow/controls": "^1.1.2",
         "@vue-flow/core": "^1.47.0",

ui/src/components/basicauth/BasicAuthSetup.vue

@@ -86,8 +86,8 @@
                                 </el-input>
                             </el-form-item>
                             <div class="password-requirements mb-4">
-                                <el-text>
-                                    8+ chars, 1 upper, 1 number
+                                <el-text>     
+                                    {{ t('setup.form.password_requirements') }}
                                 </el-text>
                             </div>
                         </el-form>
@@ -502,7 +502,7 @@
         localStorage.removeItem("basicAuthUserCreated")
         localStorage.setItem("basicAuthSetupCompletedAt", new Date().toISOString())
 
-        router.push({name: "login"})
+        router.push({name: "welcome"})
     }
 </script>
 

ui/src/components/basicauth/setup.scss

@@ -101,7 +101,7 @@ $checkbox-checked-color: #8405FF;
 
   .el-text {
     color: var(--ks-content-tertiary);
-    font-size: 14px;
+    font-size: 12px;
   }
 }
 

ui/src/components/dashboard/composables/charts.ts

@@ -3,6 +3,8 @@ import Utils from "../../../utils/utils";
 import {cssVariable, State} from "@kestra-io/ui-libs";
 import {getSchemeValue} from "../../../utils/scheme";
 
+import {useMiscStore} from "override/stores/misc";
+
 export function tooltip(tooltipModel: {
     title?: string[];
     body?: { lines: string[] }[];
@@ -115,7 +117,7 @@ export function extractState(value: any) {
     return value;
 }
 
-export function chartClick(moment: any, router: any, route: any, event: any, parsedData: any, elements: any, type = "label") {
+export function chartClick(moment: any, router: any, route: any, event: any, parsedData: any, elements: any, type = "label", filters: Record<string, any> = {}) {
     const query: Record<string, any> = {};
 
     if (elements && parsedData) {
@@ -192,7 +194,11 @@ export function chartClick(moment: any, router: any, route: any, event: any, par
             params: {
                 tenant: route.params.tenant,
             },
-            query: query,
+            query: {
+                ...query,
+                ...filters,
+                "filters[timeRange][EQUALS]":useMiscStore()?.configs?.chartDefaultDuration ?? "P30D"
+            },
         });
     }
 }

ui/src/components/dashboard/sections/TimeSeries.vue

@@ -49,6 +49,8 @@
         showDefault: {type: Boolean, default: false},
         short: {type: Boolean, default: false},
         execution: {type: Boolean, default: false},
+        flow: {type: String, default: undefined},
+        namespace: {type: String, default: undefined},
     });
 
 
@@ -153,7 +155,10 @@
                 if (data.type === "io.kestra.plugin.core.dashboard.data.Logs" || props.execution) {
                     return;
                 }
-                chartClick(moment, router, route, {}, parsedData.value, elements, "label");
+                chartClick(moment, router, route, {}, parsedData.value, elements, "label", {
+                    ...(props.namespace ? {"filters[namespace][IN]": props.namespace} : {}),
+                    ...(props.flow ? {"filters[flowId][EQUALS]": props.flow} : {})              
+                });
             },
         }, theme.value);
     });

ui/src/components/flows/Flows.vue

@@ -213,6 +213,8 @@
                                             :filters="chartFilters()"
                                             showDefault
                                             short
+                                            :flow="scope.row.id"
+                                            :namespace="scope.row.namespace"
                                         />
                                     </template>
                                 </el-table-column>

ui/src/components/inputs/PlaygroundRunTaskButton.vue

@@ -7,14 +7,14 @@
         :disabled="!playgroundStore.readyToStart"
     >
         <el-icon><Play /></el-icon>
-        <span>{{ t('playground.run_task') }}</span>
+        <span>{{ $t('playground.run_task') }}</span>
         <template #dropdown>
             <el-dropdown-menu>
                 <el-dropdown-item :icon="Play" @click="playgroundStore.runUntilTask(taskId)">
-                    {{ t('playground.run_this_task') }}
+                    {{ $t('playground.run_this_task') }}
                 </el-dropdown-item>
                 <el-dropdown-item :icon="PlayBoxMultiple" @click="playgroundStore.runUntilTask(taskId, true)">
-                    {{ t('playground.run_task_and_downstream') }}
+                    {{ $t('playground.run_task_and_downstream') }}
                 </el-dropdown-item>
             </el-dropdown-menu>
         </template>
@@ -22,12 +22,10 @@
 </template>
 
 <script setup lang="ts">
-    import {useI18n} from "vue-i18n";
     import {usePlaygroundStore} from "../../stores/playground";
     import Play from "vue-material-design-icons/Play.vue";
     import PlayBoxMultiple from "vue-material-design-icons/PlayBoxMultiple.vue";
 
-    const {t} = useI18n();
     const playgroundStore = usePlaygroundStore();
 
     defineProps<{

ui/src/components/layout/OnlyLeftMenuLayout.vue

@@ -12,6 +12,7 @@
     import {useCoreStore} from "../../stores/core";
     import {useMiscStore} from "override/stores/misc";
     import {computed, onMounted} from "vue";
+    import {useLayoutStore} from "../../stores/layout";
 
     const coreStore = useCoreStore();
     const miscStore = useMiscStore();
@@ -22,7 +23,9 @@
         document.getElementsByTagName("html")[0].classList.remove(collapse ? "menu-not-collapsed" : "menu-collapsed");
     }
 
+    const layoutStore = useLayoutStore();
+
     onMounted(() => {
-        onMenuCollapse(localStorage.getItem("menuCollapsed") === "true")
+        onMenuCollapse(Boolean(layoutStore.sideMenuCollapsed))
     });
 </script>

ui/src/components/layout/SideBar.vue

@@ -28,7 +28,7 @@
 </template>
 
 <script setup lang="ts">
-    import {onUpdated, ref, computed, h, watch} from "vue";
+    import {onUpdated, computed, h, watch} from "vue";
     import {useI18n} from "vue-i18n";
     import {useRoute} from "vue-router";
     import {useMediaQuery} from "@vueuse/core";
@@ -118,7 +118,10 @@
         ];
     });
 
-    const collapsed = ref(localStorage.getItem("menuCollapsed") === "true")
+    const collapsed = computed({
+        get: () => layoutStore.sideMenuCollapsed,
+        set: (v: boolean) => layoutStore.setSideMenuCollapsed(v),
+    })
 
     const isSmallScreen = useMediaQuery("(max-width: 768px)")
 

ui/src/main.js

@@ -25,10 +25,6 @@ const handleAuthError = (error, to) => {
 
 initApp(app, routes, null, en).then(({router, piniaStore}) => {
     router.beforeEach(async (to, from, next) => {
-        if (to.meta?.anonymous === true) {
-            return next();
-        }
-
         if(to.path === from.path && to.query === from.query) {
             return next(); // Prevent navigation if the path and query are the same
         }
@@ -45,13 +41,28 @@ initApp(app, routes, null, en).then(({router, piniaStore}) => {
                 if (validationErrors?.length > 0) {
                     // Creds exist in config but failed validation
                     // Route to login to show errors
+                    if (to.name === "login") {
+                        return next();
+                    }
+
                     return next({name: "login"})
                 } else {
                     // No creds in config - redirect to set it up
+                    if (to.name === "setup") {
+                        return next();
+                    }
+
                     return next({name: "setup"})
                 }
             }
 
+            if (to.meta?.anonymous === true) {
+                if (to.name === "setup") {
+                    return next({name: "login"});
+                }
+                return next();
+            }
+
             const hasCredentials = BasicAuth.isLoggedIn()
 
             if (!hasCredentials) {
@@ -92,6 +103,6 @@ initApp(app, routes, null, en).then(({router, piniaStore}) => {
     }, null, router, true);
 
     // mount
-    app.mount("#app")
+    router.isReady().then(() => app.mount("#app"))
 });
 

ui/src/stores/layout.ts

@@ -12,7 +12,13 @@ export const useLayoutStore = defineStore("layout", {
         topNavbar: undefined,
         envName: localStorage.getItem("envName") || undefined,
         envColor: localStorage.getItem("envColor") || undefined,
-        sideMenuCollapsed: localStorage.getItem("menuCollapsed") === "true",
+        sideMenuCollapsed: (() => {
+            if (typeof window === "undefined") {
+                return false;
+            }
+
+            return localStorage.getItem("menuCollapsed") === "true" || window.matchMedia("(max-width: 768px)").matches;
+        })(),
     }),
     getters: {},
     actions: {

ui/src/translations/de.json

@@ -1612,7 +1612,8 @@
         "email": "E-Mail",
         "firstName": "Vorname",
         "lastName": "Nachname",
-        "password": "Passwort"
+        "password": "Passwort",
+        "password_requirements": "Das Passwort muss mindestens 8 Zeichen lang sein und mindestens 1 Großbuchstaben und 1 Zahl enthalten."
       },
       "login": "Anmelden",
       "logout": "Abmelden",

ui/src/translations/en.json

@@ -1477,7 +1477,8 @@
         "email": "Email",
         "firstName": "First Name",
         "lastName": "Last Name",
-        "password": "Password"
+        "password": "Password",
+        "password_requirements": "Password must be at least 8 characters long and include at least 1 uppercase letter and 1 number."
       },
       "validation": {
         "email_required": "Email is required",

ui/src/translations/es.json

@@ -1612,7 +1612,8 @@
         "email": "Correo electrónico",
         "firstName": "Nombre",
         "lastName": "Apellido",
-        "password": "Contraseña"
+        "password": "Contraseña",
+        "password_requirements": "La contraseña debe tener al menos 8 caracteres y contener al menos 1 letra mayúscula y 1 número."
       },
       "login": "Iniciar sesión",
       "logout": "Cerrar sesión",

ui/src/translations/fr.json

@@ -1612,7 +1612,8 @@
         "email": "E-mail",
         "firstName": "Prénom",
         "lastName": "Nom de famille",
-        "password": "Mot de passe"
+        "password": "Mot de passe",
+        "password_requirements": "Le mot de passe doit comporter au moins 8 caractères, inclure au moins 1 lettre majuscule et 1 chiffre."
       },
       "login": "Connexion",
       "logout": "Déconnexion",

ui/src/translations/hi.json

@@ -1612,7 +1612,8 @@
         "email": "ईमेल",
         "firstName": "पहला नाम",
         "lastName": "अंतिम नाम",
-        "password": "पासवर्ड"
+        "password": "पासवर्ड",
+        "password_requirements": "पासवर्ड कम से कम 8 अक्षरों का होना चाहिए और इसमें कम से कम 1 बड़ा अक्षर और 1 संख्या शामिल होनी चाहिए।"
       },
       "login": "लॉगिन",
       "logout": "लॉगआउट",

ui/src/translations/it.json

@@ -1612,7 +1612,8 @@
         "email": "Email",
         "firstName": "Nome",
         "lastName": "Cognome",
-        "password": "Password"
+        "password": "Password",
+        "password_requirements": "La password deve essere lunga almeno 8 caratteri e includere almeno 1 lettera maiuscola e 1 numero."
       },
       "login": "Accedi",
       "logout": "Logout",

ui/src/translations/ja.json

@@ -1612,7 +1612,8 @@
         "email": "メール",
         "firstName": "名",
         "lastName": "姓",
-        "password": "パスワード"
+        "password": "パスワード",
+        "password_requirements": "パスワードは8文字以上で、少なくとも1つの大文字と1つの数字を含める必要があります。"
       },
       "login": "ログイン",
       "logout": "ログアウト",

ui/src/translations/ko.json

@@ -1612,7 +1612,8 @@
         "email": "이메일",
         "firstName": "이름",
         "lastName": "성씨",
-        "password": "비밀번호"
+        "password": "비밀번호",
+        "password_requirements": "비밀번호는 최소 8자 이상이어야 하며, 최소 1개의 대문자와 1개의 숫자를 포함해야 합니다."
       },
       "login": "로그인",
       "logout": "로그아웃",

ui/src/translations/pl.json

@@ -1612,7 +1612,8 @@
         "email": "Email",
         "firstName": "Imię",
         "lastName": "Nazwisko",
-        "password": "Hasło"
+        "password": "Hasło",
+        "password_requirements": "Hasło musi mieć co najmniej 8 znaków i zawierać co najmniej 1 wielką literę oraz 1 cyfrę."
       },
       "login": "Zaloguj się",
       "logout": "Wyloguj się",

ui/src/translations/pt.json

@@ -1612,7 +1612,8 @@
         "email": "Email",
         "firstName": "Nome",
         "lastName": "Sobrenome",
-        "password": "Senha"
+        "password": "Senha",
+        "password_requirements": "A senha deve ter pelo menos 8 caracteres e incluir pelo menos 1 letra maiúscula e 1 número."
       },
       "login": "Login",
       "logout": "Sair",

ui/src/translations/pt_BR.json

@@ -1612,7 +1612,8 @@
         "email": "Email",
         "firstName": "Nome",
         "lastName": "Sobrenome",
-        "password": "Senha"
+        "password": "Senha",
+        "password_requirements": "A senha deve ter pelo menos 8 caracteres e incluir pelo menos 1 letra maiúscula e 1 número."
       },
       "login": "Login",
       "logout": "Sair",

ui/src/translations/ru.json

@@ -1612,7 +1612,8 @@
         "email": "Электронная почта",
         "firstName": "Имя",
         "lastName": "Фамилия",
-        "password": "Пароль"
+        "password": "Пароль",
+        "password_requirements": "Пароль должен содержать не менее 8 символов, включая как минимум 1 заглавную букву и 1 цифру."
       },
       "login": "Войти",
       "logout": "Выход",

ui/src/translations/zh_CN.json

@@ -1612,7 +1612,8 @@
         "email": "电子邮件",
         "firstName": "名字",
         "lastName": "姓氏",
-        "password": "密码"
+        "password": "密码",
+        "password_requirements": "密码必须至少包含8个字符，并且至少包含1个大写字母和1个数字。"
       },
       "login": "登录",
       "logout": "注销",