jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(system): refactor concurrency limit to use a counter

Browse Source 
A counter allow to lock by flow which solves the race when two executions are created at the same time and the executoion_runnings table is empty.

Evaluating concurrency limit on the main executionQueue method also avoid an unexpected behavior where the CREATED execution is processed twice as its status didn't change immediatly when QUEUED.

Closes https://github.com/kestra-io/kestra-ee/issues/4877
This commit is contained in:
Loïc Mathieu
2025-10-07 16:50:09 +02:00
parent 6dea3d2a56
commit 4a9564be3c
23 changed files with 271 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
jdbc-mysql/src/main/java/io/kestra/runner/mysql/MysqlConcurrencyLimitStorage.java Normal file
View File

@@ -0,0 +1,15 @@
package io.kestra.runner.mysql;
import io.kestra.core.runners.ConcurrencyLimit;
import io.kestra.jdbc.runner.AbstractJdbcConcurrencyLimitStorage;
import io.kestra.repository.mysql.MysqlRepository;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
@Singleton
@MysqlQueueEnabled
public class MysqlConcurrencyLimitStorage extends AbstractJdbcConcurrencyLimitStorage {
public MysqlConcurrencyLimitStorage(@Named("concurrencylimit") MysqlRepository<ConcurrencyLimit> repository) {
super(repository);
}
}

15
jdbc-mysql/src/main/java/io/kestra/runner/mysql/MysqlExecutionRunningStorage.java
View File

@@ -1,15 +0,0 @@
package io.kestra.runner.mysql;
import io.kestra.core.runners.ExecutionRunning;
import io.kestra.jdbc.runner.AbstractJdbcExecutionRunningStorage;
import io.kestra.repository.mysql.MysqlRepository;
import jakarta.inject.Named;
import jakarta.inject.Singleton;
@Singleton
@MysqlQueueEnabled
public class MysqlExecutionRunningStorage extends AbstractJdbcExecutionRunningStorage {
public MysqlExecutionRunningStorage(@Named("executionrunning") MysqlRepository<ExecutionRunning> repository) {
super(repository);
}
}

8
jdbc-mysql/src/main/java/io/kestra/runner/mysql/MysqlQueueFactory.java
View File

@@ -145,14 +145,6 @@ public class MysqlQueueFactory implements QueueFactoryInterface {
return new MysqlQueue<>(SubflowExecutionEnd.class, applicationContext);
}
@Override
@Singleton
@Named(QueueFactoryInterface.EXECUTION_RUNNING_NAMED)
@Bean(preDestroy = "close")
public QueueInterface<ExecutionRunning> executionRunning() {
return new MysqlQueue<>(ExecutionRunning.class, applicationContext);
}
@Override
@Singleton
@Named(QueueFactoryInterface.MULTIPLE_CONDITION_EVENT_NAMED)

2
jdbc-mysql/src/main/resources/migrations/mysql/V1_42__purge_execution_running.sql
View File

@@ -1,2 +0,0 @@
-- We must truncate the table as in 0.24 there was a bug that lead to records not purged in this table
truncate table execution_running;

11
jdbc-mysql/src/main/resources/migrations/mysql/V1_40__execution_running.sql → jdbc-mysql/src/main/resources/migrations/mysql/V1_44__concurrency-limit.sql
View File

@@ -1,12 +1,17 @@
CREATE TABLE IF NOT EXISTS execution_running (
CREATE TABLE IF NOT EXISTS concurrency_limit (
`key` VARCHAR(250) NOT NULL PRIMARY KEY,
`value` JSON NOT NULL,
`tenant_id` VARCHAR(250) GENERATED ALWAYS AS (value ->> '$.tenantId') STORED,
`namespace` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.namespace') STORED NOT NULL,
`flow_id` VARCHAR(150) GENERATED ALWAYS AS (value ->> '$.flowId') STORED NOT NULL,
`running` INT GENERATED ALWAYS AS (value ->> '$.running') STORED NOT NULL,
INDEX ix_flow (tenant_id, namespace, flow_id)
);
DROP TABLE IF EXISTS execution_running;
DELETE FROM queues WHERE type = 'io.kestra.core.runners.ExecutionRunning';
ALTER TABLE queues MODIFY COLUMN `type` ENUM(
'io.kestra.core.models.executions.Execution',
'io.kestra.core.models.templates.Template',
@@ -24,5 +29,5 @@ ALTER TABLE queues MODIFY COLUMN `type` ENUM(
'io.kestra.core.server.ClusterEvent',
'io.kestra.core.runners.SubflowExecutionEnd',
'io.kestra.core.models.flows.FlowInterface',
'io.kestra.core.runners.ExecutionRunning'
) NOT NULL;
'io.kestra.core.runners.MultipleConditionEvent'
) NOT NULL;