import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.github.spotbugs.snom.SpotBugsTask buildscript { repositories { maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath 'com.bmuschko:gradle-docker-plugin:7.2.0' } } plugins { id 'base' id 'pmd' id 'com.diffplug.spotless' version '6.0.0' id 'com.github.hierynomus.license' version '0.16.1' id 'com.github.spotbugs' version '5.0.6' id 'version-catalog' id 'maven-publish' } apply from: "$rootDir/publish-repositories.gradle" repositories { mavenCentral() } Properties env = new Properties() rootProject.file('.env').withInputStream { env.load(it) } if (!env.containsKey('VERSION')) { throw new Exception('Version not specified in .env file...') } // `version` is used as the application build version for artifacts like jars // `image_tag` is used as the docker tag applied to built images. // These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable. // For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'. ext { version = System.getenv("VERSION") ?: env.VERSION image_tag = System.getenv("VERSION") ?: 'dev' } def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython -> /* In java, we don't have a second linter/styling tool other than spotless so it doesn't really matter if we write a newline or not for startComment/endComment. However, in python, we are using black that double-checks and reformats the code. Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a big deal and would be reformatted (removed) because of black's specs. */ def tmp = File.createTempFile('tmp', '.tmp') tmp.withWriter { def w = it if (startComment.length() > 0 || !isPython) { w.writeLine(startComment) } license.eachLine { w << lineComment w.writeLine(it) } if (endComment.length() > 0 || !isPython) { w.writeLine(endComment) } w.writeLine("") if (isPython) { w.writeLine("") } } return tmp } def createPythonLicenseWith = { license -> return createLicenseWith(license, '', '', '', true) } def createJavaLicenseWith = { license -> return createLicenseWith(license, '/*', ' */', ' * ', false) } // We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a // monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable. def createSpotlessTarget = { pattern -> def excludes = [ '.gradle', 'node_modules', '.eggs', '.mypy_cache', '.venv', '*.egg-info', 'build', 'dbt-project-template', 'dbt-project-template-mssql', 'dbt-project-template-mysql', 'dbt-project-template-oracle', 'dbt-project-template-clickhouse', 'dbt-project-template-snowflake', 'dbt_test_config', 'normalization_test_output', 'tools', 'secrets', 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. 'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad ] if (System.getenv().containsKey("SUB_BUILD")) { excludes.add("airbyte-integrations/connectors") } return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" }) } spotless { java { target createSpotlessTarget('**/*.java') importOrder() eclipse('4.21.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml')) licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) removeUnusedImports() trimTrailingWhitespace() } groovyGradle { target createSpotlessTarget('**/*.gradle') } sql { target createSpotlessTarget('**/*.sql') dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) } format 'styling', { target createSpotlessTarget(['**/*.yaml', '**/*.json']) prettier() } } check.dependsOn 'spotlessApply' @SuppressWarnings('GroovyAssignabilityCheck') def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) { return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) { def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1' def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase() def isArm64 = arch == "aarch64" || arch == "arm64" def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64' def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14' def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:1.19-alpine' : 'amd64/nginx:1.19-alpine' def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:${jdkVersion}-slim" : "amd64/openjdk:${jdkVersion}-slim" def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64' inputDir = file("$projectDir/build/docker") platform = buildPlatform images.add("airbyte/$artifactName:$buildTag") buildArgs.put('JDK_VERSION', jdkVersion) buildArgs.put('DOCKER_BUILD_ARCH', buildArch) buildArgs.put('ALPINE_IMAGE', alpineImage) buildArgs.put('NGINX_IMAGE', nginxImage) buildArgs.put('JDK_IMAGE', openjdkImage) buildArgs.put('VERSION', buildVersion) }) } @SuppressWarnings('GroovyAssignabilityCheck') def Task getPublishArtifactsTask(String buildVersion, Project subproject) { // generate a unique task name based on the directory name. return task ("publishArtifact-$subproject.name" { apply plugin: 'maven-publish' publishing { repositories { publications { "$subproject.name"(MavenPublication) { from subproject.components.java // use the subproject group and name with the assumption there are no identical subproject // names, group names or subproject group/name combination. groupId = "$subproject.group" artifactId = "$subproject.name" version = "$buildVersion" repositories.add(rootProject.repositories.getByName('cloudrepo')) } } } } }) } allprojects { apply plugin: 'com.bmuschko.docker-remote-api' task copyDocker(type: Sync) { from "${project.projectDir}/Dockerfile" into "build/docker/" } } allprojects { apply plugin: 'base' // by default gradle uses directory as the project name. That works very well in a single project environment but // projects clobber each other in an environments with subprojects when projects are in directories named identically. def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.') group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}" project.archivesBaseName = "${project.group}-${project.name}" version = rootProject.ext.version } // Java projects common configurations subprojects { if (project.name == 'airbyte-webapp' || project.name == 'airbyte-webapp-e2e-tests') { return } apply plugin: 'java' apply plugin: 'jacoco' apply plugin: 'com.github.spotbugs' apply plugin: 'pmd' sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 repositories { mavenCentral() maven { // TODO(Issue-4915): Remove this when upstream is merged in. url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' } maven { url 'https://jitpack.io' } } pmd { consoleOutput = true ignoreFailures = false rulesMinimumPriority = 5 ruleSets = [] ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml')) toolVersion = '6.43.0' } jacoco { toolVersion = "0.8.7" } jacocoTestReport { dependsOn test reports { html.required = true xml.required = true csv.required = false } } jacocoTestCoverageVerification { violationRules { failOnViolation = false rule { element = 'CLASS' excludes = ['**/*Test*', '**/generated*'] limit { counter = 'BRANCH' minimum = 0.8 } limit { counter = 'INSTRUCTION' minimum = 0.8 } } } } def integrationTagName = 'platform-integration' def slowIntegrationTagName = 'platform-slow-integration' // make tag accessible in submodules. ext { cloudStorageTestTagName = 'cloud-storage' } spotbugs { ignoreFailures = false effort = 'max' excludeFilter = rootProject.file('spotbugs-exclude-filter-file.xml') reportLevel = 'high' showProgress = false toolVersion = '4.6.0' } test { jacoco { enabled = true excludes = ['**/*Test*', '**/generated*'] } useJUnitPlatform { excludeTags(integrationTagName, slowIntegrationTagName, cloudStorageTestTagName) } testLogging() { events 'failed' exceptionFormat 'full' // showStandardStreams = true } finalizedBy jacocoTestReport } task newIntegrationTests(type: Test) { useJUnitPlatform { includeTags integrationTagName } testLogging() { events 'failed' exceptionFormat 'full' } finalizedBy jacocoTestReport } task slowIntegrationTests(type: Test) { useJUnitPlatform { includeTags slowIntegrationTagName } testLogging() { events 'failed' exceptionFormat 'full' } finalizedBy jacocoTestReport } task allTests(type: Test) { useJUnitPlatform() testLogging() { events 'failed' exceptionFormat 'full' } finalizedBy jacocoTestReport } dependencies { if (project.name != 'airbyte-commons') { implementation project(':airbyte-commons') } implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here. implementation libs.bundles.jackson implementation libs.guava implementation libs.commons.io implementation libs.bundles.apache implementation libs.slf4j.api // SLF4J as a facade over Log4j2 required dependencies implementation libs.bundles.log4j // Bridges from other logging implementations to SLF4J implementation libs.bundles.slf4j // Dependencies for logging to cloud storage, as well as the various clients used to do so. implementation libs.appender.log4j2 implementation libs.aws.java.sdk.s3 implementation libs.google.cloud.storage implementation libs.s3 // Lombok dependencies compileOnly libs.lombok annotationProcessor libs.lombok testCompileOnly libs.lombok testAnnotationProcessor libs.lombok testRuntimeOnly libs.junit.jupiter.engine testImplementation libs.bundles.junit testImplementation libs.assertj.core testImplementation libs.junit.pioneer // adds owasp plugin spotbugsPlugins libs.findsecbugs.plugin implementation libs.spotbugs.annotations } tasks.withType(Tar) { duplicatesStrategy DuplicatesStrategy.INCLUDE } tasks.withType(Zip) { duplicatesStrategy DuplicatesStrategy.INCLUDE } tasks.withType(SpotBugsTask) { // Reports can be found under each subproject in build/spotbugs/ reports { xml.required = false html.required = true } } tasks.withType(Pmd) { exclude '**/generated/**' exclude '**/jooq/**' } javadoc.options.addStringOption('Xdoclint:none', '-quiet') check.dependsOn 'jacocoTestCoverageVerification' } task('generate') { dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) } dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } } task('format') { dependsOn generate dependsOn spotlessApply dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) } } // add licenses for python projects. subprojects { def pythonFormatTask = project.tasks.findByName('blackFormat') if (pythonFormatTask != null) { apply plugin: "com.github.hierynomus.license" license { header rootProject.file("LICENSE_SHORT") } task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) source = fileTree(dir: projectDir) .include("**/*.py") .exclude(".venv/**/*.py") .exclude("**/airbyte_api_client/**/*.py") .exclude("**/__init__.py") strictCheck = true } def licenseTask = project.tasks.findByName('licenseFormatPython') blackFormat.dependsOn licenseTask isortFormat.dependsOn licenseTask flakeCheck.dependsOn licenseTask def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles') if (generateFilesTask != null) { licenseTask.dependsOn generateFilesTask } } } task('generate-docker') { dependsOn(':airbyte-bootloader:assemble') dependsOn(':airbyte-workers:assemble') dependsOn(':airbyte-webapp:assemble') dependsOn(':airbyte-server:assemble') dependsOn(':airbyte-db:db-lib:assemble') dependsOn(':airbyte-config:init:assemble') dependsOn(':airbyte-temporal:assemble') } // produce reproducible archives // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives) tasks.withType(AbstractArchiveTask) { preserveFileTimestamps = false reproducibleFileOrder = true } // definition for publishing catalog { versionCatalog { from(files("deps.toml")) } } publishing { publications { // This block is present for dependency catalog publishing. maven(MavenPublication) { groupId = 'io.airbyte' artifactId = 'oss-catalog' from components.versionCatalog // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id. // e.g. the subproject :airbyte-scheduler:scheduler-models is imported at io.airbyte.airbyte-config:config-persistence:. } } repositories.add(rootProject.repositories.getByName('cloudrepo')) }