jprdonnelly/kestra
1
0
Fork 0
mirror of https://github.com/kestra-io/kestra.git synced 2025-12-19 18:05:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(tests): add annotation and modify CI to handle flaky tests, play… (#11914)

Browse Source 
* feat(tests): add annotation and modify CI to handle flaky tests, play them but not fail

* feat(tests): add description to flaky annotation

* fix(tests): add flaky to to KV sanity check

* fix(tests): typo

* feat(tests): add a flaky test

---------

Co-authored-by: nKwiatkowski <nkwiatkowski@kestra.io>
This commit is contained in:
Nicolas K.
2025-10-10 15:29:51 +02:00
committed by GitHub
parent 00bcfac0ee
commit ff18fc40ef
10 changed files with 80 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
build.gradle
View File

@@ -206,41 +206,69 @@ subprojects {subProj ->
testImplementation 'org.assertj:assertj-core'
}
test {
useJUnitPlatform()
reports {
junitXml.required = true
junitXml.outputPerTestCase = true
junitXml.mergeReruns = true
junitXml.includeSystemErrLog = true;
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
}
def commonTestConfig = { Test t ->
// set Xmx for test workers
maxHeapSize = '4g'
t.maxHeapSize = '4g'
// configure en_US default locale for tests
systemProperty 'user.language', 'en'
systemProperty 'user.country', 'US'
t.systemProperty 'user.language', 'en'
t.systemProperty 'user.country', 'US'
environment 'SECRET_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
environment 'SECRET_NEW_LINE', "cGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2\nZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZl\neXJsb25n"
environment 'SECRET_WEBHOOK_KEY', "secretKey".bytes.encodeBase64().toString()
environment 'SECRET_NON_B64_SECRET', "some secret value"
environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
environment 'ENV_TEST1', "true"
environment 'ENV_TEST2', "Pass by env"
t.environment 'SECRET_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
t.environment 'SECRET_NEW_LINE', "cGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2\nZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZleXJsb25ncGFzc3dvcmR2ZXJ5dmVyeXZl\neXJsb25n"
t.environment 'SECRET_WEBHOOK_KEY', "secretKey".bytes.encodeBase64().toString()
t.environment 'SECRET_NON_B64_SECRET', "some secret value"
t.environment 'SECRET_PASSWORD', "cGFzc3dvcmQ="
t.environment 'ENV_TEST1', "true"
t.environment 'ENV_TEST2', "Pass by env"
if (subProj.name == 'core' || subProj.name == 'jdbc-h2' || subProj.name == 'jdbc-mysql' || subProj.name == 'jdbc-postgres') {
// JUnit 5 parallel settings
systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
t.systemProperty 'junit.jupiter.execution.parallel.enabled', 'true'
t.systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent'
t.systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'same_thread'
t.systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic'
}
}
tasks.register('flakyTest', Test) { Test t ->
group = 'verification'
description = 'Runs tests tagged @Flaky but does not fail the build.'
useJUnitPlatform {
includeTags 'flaky'
}
ignoreFailures = true
reports {
junitXml.required = true
junitXml.outputPerTestCase = true
junitXml.mergeReruns = true
junitXml.includeSystemErrLog = true
junitXml.outputLocation = layout.buildDirectory.dir("test-results/flakyTest")
}
commonTestConfig(t)
}
test {
useJUnitPlatform {
excludeTags 'flaky'
}
reports {
junitXml.required = true
junitXml.outputPerTestCase = true
junitXml.mergeReruns = true
junitXml.includeSystemErrLog = true
junitXml.outputLocation = layout.buildDirectory.dir("test-results/test")
}
commonTestConfig(it)
finalizedBy(tasks.named('flakyTest'))
}
testlogger {
theme = 'mocha-parallel'
showExceptions = true

2
core/src/test/java/io/kestra/core/runners/AbstractRunnerTest.java
View File

@@ -12,7 +12,6 @@ import io.kestra.core.queues.QueueInterface;
import io.kestra.plugin.core.flow.*;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
@@ -242,7 +241,6 @@ public abstract class AbstractRunnerTest {
multipleConditionTriggerCaseTest.flowTriggerPreconditions();
}
@Disabled
@Test
@LoadFlows(value = {"flows/valids/flow-trigger-preconditions-flow-listen.yaml",
"flows/valids/flow-trigger-preconditions-flow-a.yaml",

2
core/src/test/java/io/kestra/core/runners/ExecutionServiceTest.java
View File

@@ -20,7 +20,6 @@ import io.kestra.plugin.core.debug.Return;
import io.kestra.plugin.core.flow.Pause;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.RetryingTest;
import org.slf4j.event.Level;
@@ -218,7 +217,6 @@ class ExecutionServiceTest {
assertThat(restart.getLabels()).contains(new Label(Label.REPLAY, "true"));
}
@Disabled
@Test
@LoadFlows({"flows/valids/parallel-nested.yaml"})
void replayParallel() throws Exception {

4
core/src/test/java/io/kestra/core/tasks/test/SanityCheckTest.java
View File

@@ -1,11 +1,11 @@
package io.kestra.core.tasks.test;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.executions.Execution;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.State;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@@ -33,7 +33,7 @@ class SanityCheckTest {
assertThat(execution.getState().getCurrent()).isEqualTo(State.Type.SUCCESS);
}
@Disabled
@FlakyTest
@Test
@ExecuteFlow("sanity-checks/kv.yaml")
void qaKv(Execution execution) {

11
core/src/test/java/io/kestra/plugin/core/flow/PauseTest.java
View File

@@ -2,6 +2,7 @@ package io.kestra.plugin.core.flow;
import com.google.common.io.CharStreams;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.executions.Execution;
@@ -23,7 +24,6 @@ import io.netty.handler.codec.http.multipart.*;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.validation.ConstraintViolationException;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@@ -54,17 +54,20 @@ public class PauseTest {
suite.run(runnerUtils);
}
@Disabled("This test is too flaky and it always pass in JDBC and Kafka")
@FlakyTest(description = "This test is too flaky and it always pass in JDBC and Kafka")
@Test
void delay() throws Exception {
suite.runDelay(runnerUtils);
}
@Disabled("This test is too flaky and it always pass in JDBC and Kafka")
@FlakyTest(description = "This test is too flaky and it always pass in JDBC and Kafka")
@Test
void delayFromInput() throws Exception {
suite.runDurationFromInput(runnerUtils);
}
@Disabled("This test is too flaky and it always pass in JDBC and Kafka")
@FlakyTest(description = "This test is too flaky and it always pass in JDBC and Kafka")
@Test
void parallelDelay() throws Exception {
suite.runParallelDelay(runnerUtils);
}

2
core/src/test/java/io/kestra/plugin/core/kv/SetTest.java
View File

@@ -1,6 +1,7 @@
package io.kestra.plugin.core.kv;
import io.kestra.core.context.TestRunContextFactory;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.models.kv.KVType;
import io.kestra.core.models.property.Property;
@@ -167,6 +168,7 @@ class SetTest {
assertThat(expirationDate.isAfter(Instant.now().plus(Duration.ofMinutes(4))) && expirationDate.isBefore(Instant.now().plus(Duration.ofMinutes(6)))).isTrue();
}
@FlakyTest
@Test
void shouldFailGivenExistingKeyAndOverwriteFalse() throws Exception {
// Given

3
jdbc/src/test/java/io/kestra/jdbc/runner/JdbcServiceLivenessCoordinatorTest.java
View File

@@ -97,7 +97,6 @@ public abstract class JdbcServiceLivenessCoordinatorTest {
}
@FlakyTest
@Disabled
@Test
void shouldReEmitTasksWhenWorkerIsDetectedAsNonResponding() throws Exception {
CountDownLatch runningLatch = new CountDownLatch(1);
@@ -210,7 +209,7 @@ public abstract class JdbcServiceLivenessCoordinatorTest {
assertThat(receive.blockLast().getTaskRun().getState().getCurrent()).isNotEqualTo(Type.SUCCESS);
}
@Disabled
@FlakyTest
@Test
void shouldReEmitTriggerWhenWorkerIsDetectedAsNonResponding() throws Exception {
Worker worker = applicationContext.createBean(TestMethodScopedWorker.class, IdUtils.create(), 1, null);

6
scheduler/src/test/java/io/kestra/scheduler/SchedulerScheduleTest.java
View File

@@ -1,6 +1,7 @@
package io.kestra.scheduler;
import com.devskiller.friendly_id.FriendlyId;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.models.executions.TaskRun;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.flows.PluginDefault;
@@ -25,7 +26,6 @@ import io.kestra.core.utils.Await;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestInstance.Lifecycle;
@@ -545,8 +545,8 @@ public class SchedulerScheduleTest extends AbstractSchedulerTest {
}
}
@FlakyTest(description = "too flaky on CI")
@Test
@Disabled("too flaky on CI")
void recoverLASTLongRunningExecution() throws Exception {
// mock flow listeners
FlowListeners flowListenersServiceSpy = spy(this.flowListenersService);
@@ -618,8 +618,8 @@ public class SchedulerScheduleTest extends AbstractSchedulerTest {
}
}
@FlakyTest(description = "too flaky on CI")
@Test
@Disabled("too flaky on CI")
void recoverNONELongRunningExecution() throws Exception {
// mock flow listeners
FlowListeners flowListenersServiceSpy = spy(this.flowListenersService);

11
tests/src/main/java/io/kestra/core/junit/annotations/FlakyTest.java
View File

@@ -4,11 +4,18 @@ import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.junit.jupiter.api.Tag;
/**
* used to document that a test is flaky and needs to be reworked
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.SOURCE)
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Tag("flaky")
public @interface FlakyTest {
/**
* Use to explain why the test is flaky
*/
String description() default "";
}

6
webserver/src/test/java/io/kestra/webserver/controllers/api/ExecutionControllerRunnerTest.java
View File

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.ImmutableMap;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.junit.annotations.ExecuteFlow;
import io.kestra.core.junit.annotations.FlakyTest;
import io.kestra.core.junit.annotations.KestraTest;
import io.kestra.core.junit.annotations.LoadFlows;
import io.kestra.core.models.Label;
@@ -1287,9 +1288,10 @@ class ExecutionControllerRunnerTest {
assertThat(executions.getTotal()).isEqualTo(4L);
}
@RetryingTest(5)
@FlakyTest
@Test
@LoadFlows({"flows/valids/pause-test.yaml"})
void killExecutionPaused() throws TimeoutException, InterruptedException, QueueException {
void killExecutionPaused() throws TimeoutException, QueueException {
// Run execution until it is paused
Execution pausedExecution = runnerUtils.runOneUntilPaused(TENANT_ID, TESTS_FLOW_NS, "pause-test");
assertThat(pausedExecution.getState().isPaused()).isTrue();