jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(dashboard): calculate running execution duration on the fly

Browse Source
This commit is contained in:
Roman Acevedo
2025-11-25 18:05:15 +01:00
parent 64899f3103
commit 7aca309be5
17 changed files with 261 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
core/src/main/java/io/kestra/core/models/flows/State.java
View File

@@ -84,16 +84,26 @@ public class State {
);
}
/**
* non-terminated execution duration is hard to provide in SQL, so we set it to null when endDate is empty
*/
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
public Duration getDuration() {
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Optional<Duration> getDuration() {
if (this.getEndDate().isPresent()) {
return Duration.between(this.getStartDate(), this.getEndDate().get());
return Optional.of(Duration.between(this.getStartDate(), this.getEndDate().get()));
} else {
// return Duration.between(this.getStartDate(), Instant.now()); TODO improve
return null;
return Optional.empty();
}
}
/**
* @return either the Duration persisted in database, or calculate it on the fly for non-terminated executions
*/
public Duration getDurationOrComputeIt() {
return this.getDuration().orElseGet(() -> Duration.between(this.getStartDate(), Instant.now()));
}
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
public Instant getStartDate() {
return this.histories.getFirst().getDate();
@@ -111,7 +121,7 @@ public class State {
public String humanDuration() {
try {
return DurationFormatUtils.formatDurationHMS(getDuration().toMillis());
return DurationFormatUtils.formatDurationHMS(getDurationOrComputeIt().toMillis());
} catch (Throwable e) {
return getDuration().toString();
}

7
core/src/test/java/io/kestra/core/models/executions/StateDurationTest.java
View File

@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
@@ -25,7 +26,7 @@ public class StateDurationTest {
new State.History(State.Type.CREATED, ONE)
)
);
assertThat(state.getDuration()).isCloseTo(Duration.between(ONE, NOW), Duration.ofMinutes(10));
assertThat(state.getDuration()).isEmpty();
}
@Test
@@ -38,7 +39,7 @@ public class StateDurationTest {
new State.History(State.Type.SUCCESS, THREE)
)
);
assertThat(state.getDuration()).isEqualTo(Duration.between(ONE, THREE));
assertThat(state.getDuration()).isEqualTo(Optional.of(Duration.between(ONE, THREE)));
}
@Test
@@ -50,6 +51,6 @@ public class StateDurationTest {
new State.History(State.Type.RUNNING, TWO)
)
);
assertThat(state.getDuration()).isCloseTo(Duration.between(ONE, NOW), Duration.ofMinutes(10));
assertThat(state.getDuration()).isEmpty();
}
}

4
executor/src/main/java/io/kestra/executor/ExecutorService.java
View File

@@ -437,7 +437,7 @@ public class ExecutorService {
metricRegistry
.timer(MetricRegistry.METRIC_EXECUTOR_EXECUTION_DURATION, MetricRegistry.METRIC_EXECUTOR_EXECUTION_DURATION_DESCRIPTION, metricRegistry.tags(newExecution))
.record(newExecution.getState().getDuration());
.record(newExecution.getState().getDurationOrComputeIt());
return executor.withExecution(newExecution, "onEnd");
}
@@ -1170,7 +1170,7 @@ public class ExecutorService {
MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_DURATION_DESCRIPTION,
metricRegistry.tags(workerTaskResult)
)
.record(taskRun.getState().getDuration());
.record(taskRun.getState().getDurationOrComputeIt());
}
}

6
jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java
View File

@@ -624,7 +624,7 @@ public abstract class AbstractJdbcExecutionRepository extends AbstractJdbcCrudRe
}
private DailyExecutionStatistics dailyExecutionStatisticsMap(Instant date, List<ExecutionStatistics> result, String groupByType) {
long durationSum = result.stream().map(ExecutionStatistics::getDurationSum).mapToLong(value -> value).sum();
long durationSum = result.stream().map(ExecutionStatistics::getDurationSum).mapToLong(value -> value != null ? value : 0).sum();
long count = result.stream().map(ExecutionStatistics::getCount).mapToLong(value -> value).sum();
DailyExecutionStatistics build = DailyExecutionStatistics.builder()
@@ -632,8 +632,8 @@ public abstract class AbstractJdbcExecutionRepository extends AbstractJdbcCrudRe
.groupBy(groupByType)
.duration(DailyExecutionStatistics.Duration.builder()
.avg(Duration.ofMillis(durationSum / count))
.min(result.stream().map(ExecutionStatistics::getDurationMin).min(Long::compare).map(Duration::ofMillis).orElse(null))
.max(result.stream().map(ExecutionStatistics::getDurationMax).max(Long::compare).map(Duration::ofMillis).orElse(null))
.min(result.stream().map(ExecutionStatistics::getDurationMin).map(x -> x != null ? x : 0).min(Long::compare).map(Duration::ofMillis).orElse(null))
.max(result.stream().map(ExecutionStatistics::getDurationMax).map(x -> x != null ? x : 0).max(Long::compare).map(Duration::ofMillis).orElse(null))
.sum(Duration.ofMillis(durationSum))
.count(count)
.build()

2
jdbc/src/main/java/io/kestra/jdbc/runner/JdbcExecutor.java
View File

@@ -895,7 +895,7 @@ public class JdbcExecutor implements ExecutorInterface {
metricRegistry
.timer(MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_DURATION, MetricRegistry.METRIC_EXECUTOR_TASKRUN_ENDED_DURATION_DESCRIPTION, metricRegistry.tags(message))
.record(taskRun.getState().getDuration());
.record(taskRun.getState().getDurationOrComputeIt());
log.trace("TaskRun terminated: {}", taskRun);
}

4
jdbc/src/test/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepositoryTest.java
View File

@@ -1,8 +1,4 @@
package io.kestra.jdbc.repository;
public abstract class AbstractJdbcExecutionRepositoryTest extends io.kestra.core.repositories.AbstractExecutionRepositoryTest {
@Override
protected void fetchData() {
// TODO Remove the override once JDBC implementation has the QueryBuilder working
}
}

4
ui/src/components/dashboard/components/PreviewDashboardWrapper.vue
View File

@@ -12,7 +12,7 @@
<script lang="ts" setup>
import {ref, watch} from "vue";
import Sections from "../sections/Sections.vue";
import {Chart} from "../composables/useDashboards";
import {Chart} from "../types.ts";
import {useDashboardStore} from "../../../stores/dashboard";
import * as YAML_UTILS from "@kestra-io/ui-libs/flow-yaml-utils";
import throttle from "lodash/throttle";
@@ -44,7 +44,7 @@
, {immediate: true}
);
async function loadChart(chart: any) {
const yamlChart = YAML_UTILS.stringify(chart);

77
ui/src/components/dashboard/composables/useDashboards.ts
View File

@@ -9,62 +9,6 @@ import {useI18n} from "vue-i18n";
import {decodeSearchParams} from "../../filter/utils/helpers";
export type Dashboard = {
id: string;
charts: Chart[];
title?: string;
sourceCode?: string;
[key: string]: unknown;
};
export type Chart = {
id: string;
type: string;
chartOptions?: {
displayName?: string;
description?: string;
width?: number;
pagination?: {
enabled?: boolean;
[key: string]: unknown;
};
legend?:{
enabled?: boolean;
};
column: string;
[key: string]: unknown;
};
data?: {
columns?: {
[key: string]: Record<string, any>;
};
[key: string]: unknown;
};
content?: string;
source?: {
type?: string;
content?: string;
[key: string]: unknown;
};
[key: string]: unknown;
};
export type Request = {
chart: Chart["content"];
globalFilter?: Parameters;
};
export type Parameters = {
pageNumber?: number;
pageSize?: number;
startDate?: Date;
endDate?: Date;
namespace?: string;
labels?: Record<string, string>;
filters?: Record<string, any>;
};
export const ALLOWED_CREATION_ROUTES = ["home", "flows/update", "namespaces/update"];
export const STORAGE_KEYS = (params: RouteParams) => {
@@ -95,22 +39,11 @@ export function getDashboard(route: RouteLocation, type: "key" | "id"): string |
return type === "key" ? storageKey : localStorage.getItem(storageKey) || "default";
};
import Bar from "../sections/Bar.vue";
import KPI from "../sections/KPI.vue";
import Markdown from "../sections/Markdown.vue";
import Pie from "../sections/Pie.vue";
import Table from "../sections/Table.vue";
import TimeSeries from "../sections/TimeSeries.vue";
import {FilterObject} from "../../../utils/filters";
export const TYPES: Record<string, any> = {
"io.kestra.plugin.core.dashboard.chart.Bar": Bar,
"io.kestra.plugin.core.dashboard.chart.KPI": KPI,
"io.kestra.plugin.core.dashboard.chart.Markdown": Markdown,
"io.kestra.plugin.core.dashboard.chart.Pie": Pie,
"io.kestra.plugin.core.dashboard.chart.Table": Table,
"io.kestra.plugin.core.dashboard.chart.TimeSeries": TimeSeries,
};
import {FilterObject} from "../../../utils/filters";
import {Chart, Parameters, Request} from "../types.ts";
export const isKPIChart = (type: string): boolean => type === "io.kestra.plugin.core.dashboard.chart.KPI";
@@ -159,3 +92,5 @@ export function useChartGenerator(props: {chart: Chart; filters: FilterObject[];
return {percentageShown, EMPTY_TEXT, data, generate};
}
export * from "../types";

15
ui/src/components/dashboard/dashboard-types.ts Normal file
View File

@@ -0,0 +1,15 @@
import Bar from "./sections/Bar.vue";
import KPI from "./sections/KPI.vue";
import Markdown from "./sections/Markdown.vue";
import Pie from "./sections/Pie.vue";
import Table from "./sections/Table.vue";
import TimeSeries from "./sections/TimeSeries.vue";
export const TYPES: Record<string, any> = {
"io.kestra.plugin.core.dashboard.chart.Bar": Bar,
"io.kestra.plugin.core.dashboard.chart.KPI": KPI,
"io.kestra.plugin.core.dashboard.chart.Markdown": Markdown,
"io.kestra.plugin.core.dashboard.chart.Pie": Pie,
"io.kestra.plugin.core.dashboard.chart.Table": Table,
"io.kestra.plugin.core.dashboard.chart.TimeSeries": TimeSeries,
};

9
ui/src/components/dashboard/sections/Sections.vue
View File

@@ -74,7 +74,8 @@
import {ref, computed} from "vue";
import type {Dashboard, Chart} from "../composables/useDashboards";
import {TYPES, isKPIChart, isTableChart, getChartTitle} from "../composables/useDashboards";
import {isKPIChart, isTableChart, getChartTitle} from "../composables/useDashboards";
import {TYPES} from "../dashboard-types";
import {useRoute} from "vue-router";
const route = useRoute();
@@ -110,11 +111,11 @@
title: getChartTitle(chart),
description: chart?.chartOptions?.description,
});
// Make the overview of flows/dashboard/namespace specific
const filters = computed(() => {
const baseFilters: { field: string; operation: string; value: string | string[] }[] = [];
if (route.name === "flows/update") {
baseFilters.push({field: "namespace", operation: "EQUALS", value: route.params.namespace as string});
baseFilters.push({field: "flowId", operation: "EQUALS", value: route.params.id as string});
@@ -177,7 +178,7 @@ section#charts {
grid-column: span #{$i};
}
}
@for $i from 4 through 12 {
.dash-width-#{$i} {
grid-column: span 3;

8
ui/src/components/dashboard/sections/Table.vue
View File

@@ -1,5 +1,5 @@
<template>
<section v-if="data" id="table">
<section v-if="data?.results?.length" id="table">
<el-table
:id="containerID"
:data="data.results"
@@ -39,7 +39,7 @@
import type {RouteLocation} from "vue-router";
import type {Chart} from "../composables/useDashboards";
import type {Chart} from "../types.ts";
import {getDashboard, isPaginationEnabled, useChartGenerator} from "../composables/useDashboards";
import Date from "./table/columns/Date.vue";
@@ -88,11 +88,11 @@
return {field: row[key]};
case "STATE":
return {
size: "small",
size: "small",
status: row[key].toString(),
};
case "DURATION":
return {field: row[key]};
return {field: row[key], startDate: row["start_date"]};
default:
if (field.toLowerCase().includes("date")) {
return {field: row[key]};

3
ui/src/components/dashboard/sections/table/columns/Date.vue
View File

@@ -14,7 +14,8 @@
},
});
const format = localStorage.getItem(storageKeys.DATE_FORMAT_STORAGE_KEY) ?? "llll";
const momentLongDateFormat = "llll";
const format = localStorage.getItem(storageKeys.DATE_FORMAT_STORAGE_KEY) ?? momentLongDateFormat;
const formatDateIfPresent = (rawDate: string|undefined) => {
if(rawDate){
// moment(date) always return a Moment, if the date is undefined, it will return current date, we don't want that here

17
ui/src/components/dashboard/sections/table/columns/Duration.vue
View File

@@ -1,14 +1,17 @@
<template>
<span>{{ Utils.humanDuration(props.field) }}</span>
<span v-if="field">{{ Utils.humanDuration(calculatedField) }}</span>
<em v-else>{{ Utils.humanDuration(calculatedField) }}</em>
</template>
<script setup lang="ts">
import {Utils} from "@kestra-io/ui-libs";
import {computed} from "vue";
const props = defineProps({
field: {
type: Number,
required: true,
},
});
const props = defineProps<{
field: number | undefined,
startDate?: string
}>();
// handle case where execution is non-terminated, then there is no duration, we calculate it live to display it to the user
const calculatedField = computed(() => props.field === undefined && props.startDate ? ((+new Date() - new Date(props.startDate).getTime()) / 1000 ) : props.field ?? 0);
</script>

55
ui/src/components/dashboard/types.ts Normal file
View File

@@ -0,0 +1,55 @@
export type Dashboard = {
id: string;
charts: Chart[];
title?: string;
sourceCode?: string;
[key: string]: unknown;
};
export type Chart = {
id: string;
type: string;
chartOptions?: {
displayName?: string;
description?: string;
width?: number;
pagination?: {
enabled?: boolean;
[key: string]: unknown;
};
legend?:{
enabled?: boolean;
};
column: string;
[key: string]: unknown;
};
data?: {
columns?: {
[key: string]: Record<string, any>;
};
[key: string]: unknown;
};
content?: string;
source?: {
type?: string;
content?: string;
[key: string]: unknown;
};
[key: string]: unknown;
};
export type Request = {
chart: Chart["content"];
globalFilter?: Parameters;
};
export type Parameters = {
pageNumber?: number;
pageSize?: number;
startDate?: Date;
endDate?: Date;
namespace?: string;
labels?: Record<string, string>;
filters?: Record<string, any>;
};

16
ui/src/components/executions/Executions.vue
View File

@@ -198,10 +198,7 @@
<DateAgo :inverted="true" :date="scope.row?.state?.endDate" />
</template>
<template v-else-if="col.prop === 'state.duration'">
<span v-if="isRunning(scope.row)">{{
humanizeDuration(durationFrom(scope.row).toString())
}}</span>
<span v-else>{{ humanizeDuration(scope.row?.state?.duration) }}</span>
<Duration :field="scope.row?.state?.duration" :startDate="scope.row?.state?.startDate" />
</template>
<template v-else-if="col.prop === 'namespace' && $route.name !== 'flows/update'">
<span :title="invisibleSpace(scope.row?.namespace)">{{ invisibleSpace(scope.row?.namespace) }}</span>
@@ -430,8 +427,9 @@
import {filterValidLabels} from "./utils";
import {useToast} from "../../utils/toast";
import {storageKeys} from "../../utils/constants";
import {humanizeDuration, invisibleSpace} from "../../utils/filters";
import {invisibleSpace} from "../../utils/filters";
import Utils from "../../utils/utils";
import Duration from "../../components/dashboard/sections/table/columns/Duration.vue";
import action from "../../models/action";
import permission from "../../models/permission";
@@ -744,10 +742,6 @@
load(onDataLoaded);
};
const isRunning = (item: any) => {
return State.isRunning(item?.state?.current);
};
const loadQuery = (base: any) => {
let queryFilter = queryWithFilter();
@@ -767,10 +761,6 @@
return _merge(base, queryFilter);
};
const durationFrom = (item: any) => {
return +new Date() - new Date(item?.state?.startDate).getTime();
};
const genericConfirmAction = (message: string, queryAction: string, byIdAction: string, success: string, showCancelButton = true) => {
toast.confirm(
t(message, {"executionCount": queryBulkAction.value ? executionsStore.total : selection.value.length}),

133
ui/tests/storybook/components/dashboard/sections/Table.stories.tsx Normal file
View File

@@ -0,0 +1,133 @@
import Table from "../../../../../src/components/dashboard/sections/Table.vue";
import type {Chart} from "../../../../../src/components/dashboard/types.ts";
import type {Meta, StoryObj} from "@storybook/vue3-vite";
import {vueRouter} from "storybook-vue3-router";
import {useAxios} from "../../../../../src/utils/axios.ts";
import {expect, within} from "storybook/test";
const meta: Meta<typeof Table> = {
title: "Dashboard/Sections/Table",
component: Table,
decorators: [
vueRouter([
{
path: "/",
component: () => <div></div>
},
{
path: "/flows/update",
name: "flows/update",
component: () => <div></div>
},
{
path: "/executions/update",
name: "executions/update",
component: () => <div></div>
},
{
path: "/namespaces/update",
name: "namespaces/update",
component: () => <div></div>
},
])
]
}
export default meta;
export const SimpleExecutionsCase: StoryObj<typeof Table> = {
render: () => ({
setup() {
const store = useAxios() as any;
store.post = async function (uri: string) {
console.log("post request", uri)
if (uri.includes("charts/executions_finished")) {
console.log("match charts/executions_finished", uri)
return {
data: {
results: [
{
"namespace": "company.team",
"id": "2wJlDoXRsMc7jXJfQUWTE7",
"state": "RUNNING",
"flow": "sleep",
"start_date": "2025-11-25T09:28:00.000+00:00",
},
{
"namespace": "company.team",
"id": "2yiYHSqLwNbocm9FB8qK5L",
"state": "RUNNING",
"flow": "sleep",
"start_date": "2025-11-25T09:28:00.000+00:00"
},
{
"duration": 6,
"namespace": "company.team",
"id": "2Iq5tjur4bB9fRYYazstV4",
"state": "SUCCESS",
"flow": "sleep",
"end_date": "2025-11-25T09:27:00.000+00:00",
"start_date": "2025-11-25T09:27:00.000+00:00",
},
{
"namespace": "company.team",
"id": "69d95APmpdw94OkaMduCep",
"state": "RUNNING",
"flow": "sleep",
"start_date": "2025-11-25T09:27:00.000+00:00"
}
],
total: 4
}
}
}
return {results: []}
}
const chart: Chart = {
"id": "executions_finished",
"type": "io.kestra.plugin.core.dashboard.chart.Table",
"chartOptions": {
"displayName": "Executions Finished",
"width": 12,
"header": {"enabled": true},
"pagination": {"enabled": true}
},
"data": {
"columns": {
"id": {"field": "ID", "displayName": "Execution ID", "columnAlignment": "LEFT"},
"flow": {"field": "FLOW_ID", "displayName": "Flow", "columnAlignment": "LEFT"},
"state": {"field": "STATE", "displayName": "State", "columnAlignment": "LEFT"},
"duration": {"field": "DURATION", "displayName": "Duration", "columnAlignment": "LEFT"},
"end_date": {"field": "END_DATE", "displayName": "End date", "columnAlignment": "LEFT"},
"namespace": {
"field": "NAMESPACE",
"displayName": "Namespace",
"columnAlignment": "LEFT"
},
"start_date": {
"field": "START_DATE",
"displayName": "Start date",
"columnAlignment": "LEFT"
}
}
},
} as any;
return () => (
<div style="padding: 20px; background: #f5f5f5; border-radius: 8px;">
<Table chart={chart}/>
</div>
);
}
}),
async play({canvasElement}) {
const canvas = within(canvasElement);
await expect(await canvas.findByText("2wJlDoXR")).toBeVisible();
await expect(await canvas.findByText("2yiYHSqL")).toBeVisible();
await expect(await canvas.findByText("2Iq5tjur")).toBeVisible();
await expect(await canvas.findByText("69d95APm")).toBeVisible();
}
}

2
worker/src/main/java/io/kestra/worker/DefaultWorker.java
View File

@@ -854,7 +854,7 @@ public class DefaultWorker implements Worker {
metricRegistry
.timer(MetricRegistry.METRIC_WORKER_ENDED_DURATION, MetricRegistry.METRIC_WORKER_ENDED_DURATION_DESCRIPTION, metricRegistry.tags(workerTask, workerGroup))
.record(workerTask.getTaskRun().getState().getDuration());
.record(workerTask.getTaskRun().getState().getDurationOrComputeIt());
Logs.logTaskRun(
workerTask.getTaskRun(),