diff --git a/build.gradle b/build.gradle index 1e9e7276ef..4f260cc8e6 100644 --- a/build.gradle +++ b/build.gradle @@ -171,13 +171,22 @@ allprojects { subprojects {subProj -> if (subProj.name != 'platform' && subProj.name != 'jmh-benchmarks') { + apply plugin: "com.adarshr.test-logger" + apply plugin: 'jacoco' java { sourceCompatibility = targetJavaVersion targetCompatibility = targetJavaVersion } + configurations { + agent { + canBeResolved = true + canBeConsumed = true + } + } + dependencies { // Platform testAnnotationProcessor enforcedPlatform(project(":platform")) @@ -204,9 +213,49 @@ subprojects {subProj -> //assertj testImplementation 'org.assertj:assertj-core' + + agent "org.aspectj:aspectjweaver:1.9.25.1" + + testImplementation platform("io.qameta.allure:allure-bom") + testImplementation "io.qameta.allure:allure-junit5" } def commonTestConfig = { Test t -> + // BEGIN test failure configuration + // what is configured: + // - each test task will be marked as success all the time (except in case of gradle unhandled error or JVM crash) + // - a test task with failed tests will output a file in test-failures directory, so we can use that output later to check if any tests failed + t.ignoreFailures(true) + t.ext.testFailures = 0 + + // Persist failure state across runs so a previously-failing Test task can't be UP-TO-DATE and let :check pass. + def testFailureFileForThisProject = t.project.layout.buildDirectory.file("test-failures/${t.name}.failed").get().asFile + t.outputs.file(testFailureFileForThisProject) + t.outputs.upToDateWhen { !testFailureFileForThisProject.exists() } + + t.afterSuite { desc, result -> + if (desc.getParent() == null) { + def failed = (int) result.getFailedTestCount() + t.ext.testFailures += failed + + // Persist failure state for next runs + if (failed > 0) { + testFailureFileForThisProject.parentFile.mkdirs() + testFailureFileForThisProject.text = String.valueOf(failed) + } else { + if (testFailureFileForThisProject.exists()) { + testFailureFileForThisProject.delete() + } + } + } + } + t.doLast() { + if(t.ext.testFailures > 0){ + logger.error(" ${t.getProject()} ${t.getName()} had ${t.ext.testFailures} failed tests") + } + } + // END test failure configuration + // set Xmx for test workers t.maxHeapSize = '4g' @@ -232,6 +281,52 @@ subprojects {subProj -> // } } + tasks.register('integrationTest', Test) { Test t -> + description = 'Runs integration tests' + group = 'verification' + + useJUnitPlatform { + includeTags 'integration' + } + + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + + reports { + junitXml.required = true + junitXml.outputPerTestCase = true + junitXml.mergeReruns = true + junitXml.includeSystemErrLog = true + junitXml.outputLocation = layout.buildDirectory.dir("test-results/integrationTest") + } + + // Integration tests typically not parallel (but you can enable) + maxParallelForks = 1 + commonTestConfig(t) + } + + tasks.register('unitTest', Test) { Test t -> + description = 'Runs unit tests' + group = 'verification' + + useJUnitPlatform { + excludeTags 'flaky', 'integration' + } + + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + + reports { + junitXml.required = true + junitXml.outputPerTestCase = true + junitXml.mergeReruns = true + junitXml.includeSystemErrLog = true + junitXml.outputLocation = layout.buildDirectory.dir("test-results/test") + } + + commonTestConfig(t) + } + tasks.register('flakyTest', Test) { Test t -> group = 'verification' description = 'Runs tests tagged @Flaky but does not fail the build.' @@ -239,7 +334,6 @@ subprojects {subProj -> useJUnitPlatform { includeTags 'flaky' } - ignoreFailures = true reports { junitXml.required = true @@ -249,10 +343,15 @@ subprojects {subProj -> junitXml.outputLocation = layout.buildDirectory.dir("test-results/flakyTest") } commonTestConfig(t) - + // Ensure flaky tests run after test, even when Gradle is run with --parallel + mustRunAfter(tasks.named('test')) } - test { + // test task (default) + tasks.named('test', Test) { Test t -> + group = 'verification' + description = 'Runs unit tests.' + useJUnitPlatform { excludeTags 'flaky' } @@ -263,10 +362,37 @@ subprojects {subProj -> junitXml.includeSystemErrLog = true junitXml.outputLocation = layout.buildDirectory.dir("test-results/test") } - commonTestConfig(it) + commonTestConfig(t) + jvmArgs = ["-javaagent:${configurations.agent.singleFile}"] + // JUnit 5 parallel settings - CLI module does assertions on stout (mostly). + if (subProj.name != 'cli') { + systemProperty("junit.jupiter.execution.parallel.enabled", "true") + systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") + systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") + } + } - finalizedBy(tasks.named('flakyTest')) + tasks.named('check') { + // this task will aggregate test tasks results, and fail is there was any error + dependsOn(tasks.named('test'))// default behaviour + dependsOn(tasks.named('flakyTest')) + + doLast { + // Fail this subproject's :check if any of the test tasks had failures + // a test task with failed tests will be marked as SUCCESS in gradle, but will output a file in test-failures to indicate there is failed tests + def testFailuresDir = layout.buildDirectory.dir('test-failures').get().asFile + def testFailureFiles = null + if(testFailuresDir.exists()){ + testFailureFiles = testFailuresDir.listFiles({ f -> f.name.endsWith('.failed') && !f.name.contains('flakyTest')} as FileFilter) + } + + if (testFailureFiles != null && testFailureFiles.length > 0) { + def details = testFailureFiles.collect { m -> "${m.name.replace('.failed','')}: ${m.text.trim()}" }.join(', ') + throw new GradleException("marking ${project.path}:check as failed because tests failed (${details})") + } + } + finalizedBy jacocoTestReport } testlogger { @@ -359,6 +485,7 @@ subprojects { tasks.named('check') { dependsOn tasks.named('testCodeCoverageReport', JacocoReport) + finalizedBy jacocoTestReport } tasks.named('testCodeCoverageReport') { diff --git a/tests/src/main/java/io/kestra/core/junit/annotations/KestraTest.java b/tests/src/main/java/io/kestra/core/junit/annotations/KestraTest.java index f3cfa29bc5..36c70852cc 100644 --- a/tests/src/main/java/io/kestra/core/junit/annotations/KestraTest.java +++ b/tests/src/main/java/io/kestra/core/junit/annotations/KestraTest.java @@ -7,10 +7,12 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.test.annotation.TransactionMode; import io.micronaut.test.condition.TestActiveCondition; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.extension.ExtendWith; import java.lang.annotation.*; +@Tag("integration") @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.METHOD, ElementType.ANNOTATION_TYPE, ElementType.TYPE}) @ExtendWith(KestraTestExtension.class)