airbyte/build.gradle

import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
import com.github.spotbugs.snom.SpotBugsTask

// The buildscript block defines dependencies in order for .gradle file evaluation.
// This is separate from application dependencies.
// See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle.
buildscript {
    repositories {
        maven {
            url "https://plugins.gradle.org/m2/"
        }
    }
    dependencies {
        classpath 'com.bmuschko:gradle-docker-plugin:8.0.0'
        // 6.x version of OpenApi generator is only compatible with jackson-core 2.13.x onwards.
        // This conflicts with the jackson depencneis the bmuschko plugin is pulling in.
        // Since api generation is only used in the airbyte-api module and the base gradle files
        // are loaded in first, Gradle is not able to intelligently resolve this before loading in
        // the bmuschko plugin and thus placing an older jackson version on the class path.
        // The alternative is to import the openapi plugin for all modules.
        // This might need to be updated when we change openapi plugin versions.
        classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0'
    }
}

plugins {
    id 'base'
    id 'pmd'
    id 'com.diffplug.spotless' version '6.7.1'
    id 'com.github.hierynomus.license' version '0.16.1'
    id 'com.github.spotbugs' version '5.0.13'
    // The distribution plugin has been added to address the an issue with the copyGeneratedTar
    // task depending on "distTar". When that dependency has been refactored away, this plugin
    // can be removed.
    id 'distribution'
    id 'version-catalog'
    id 'maven-publish'
}

apply from: "$rootDir/publish-repositories.gradle"

repositories {
    mavenCentral()
}

Properties env = new Properties()
rootProject.file('.env').withInputStream { env.load(it) }

if (!env.containsKey('VERSION')) {
    throw new Exception('Version not specified in .env file...')
}

// `version` is used as the application build version for artifacts like jars
// `image_tag` is used as the docker tag applied to built images.
// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
ext {
    version = System.getenv("VERSION") ?: env.VERSION
    image_tag = System.getenv("VERSION") ?: 'dev'
}

def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
    /*
    In java, we don't have a second linter/styling tool other than spotless so it doesn't really
    matter if we write a newline or not for startComment/endComment.

    However, in python, we are using black that double-checks and reformats the code.
    Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
    big deal and would be reformatted (removed) because of black's specs.
    */
    def tmp = File.createTempFile('tmp', '.tmp')
    tmp.withWriter {
        def w = it
        if (startComment.length() > 0 || !isPython) {
            w.writeLine(startComment)
        }
        license.eachLine {
            w << lineComment
            w.writeLine(it)
        }
        if (endComment.length() > 0 || !isPython) {
            w.writeLine(endComment)
        }
        w.writeLine("")
        if (isPython) {
            w.writeLine("")
        }
    }
    return tmp
}

def createPythonLicenseWith = { license ->
    return createLicenseWith(license, '', '', '', true)
}

def createJavaLicenseWith = { license ->
    return createLicenseWith(license, '/*', ' */', ' * ', false)
}

// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable.
def createSpotlessTarget = { pattern ->
    def excludes = [
            '.gradle',
            'node_modules',
            '.eggs',
            '.mypy_cache',
            '.venv',
            '*.egg-info',
            'build',
            'dbt-project-template',
            'dbt-project-template-mssql',
            'dbt-project-template-mysql',
            'dbt-project-template-oracle',
            'dbt-project-template-clickhouse',
            'dbt-project-template-snowflake',
            'dbt-project-template-tidb',
            'dbt_test_config',
            'normalization_test_output',
            'tools',
            'secrets',
            'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
            'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file.
            'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows
            'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad
            'airbyte-webapp', // The webapp module uses its own auto-formatter, so spotless is not necessary here
            'airbyte-webapp-e2e-tests', // This module also uses its own auto-formatter
            'airbyte-connector-builder/connector_builder/generated', // autogenerated code doesn't need to be formatted
    ]

    if (System.getenv().containsKey("SUB_BUILD")) {
        excludes.add("airbyte-integrations/connectors")
    }

    return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" })
}

spotless {
    java {
        target createSpotlessTarget('**/*.java')

        importOrder()

        eclipse('4.21.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))

        licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
        removeUnusedImports()
        trimTrailingWhitespace()
    }
    groovyGradle {
        target createSpotlessTarget('**/*.gradle')
    }
    sql {
        target createSpotlessTarget('**/*.sql')

        dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
    }
    format 'styling', {
        target createSpotlessTarget(['**/*.yaml', '**/*.json'])

        prettier()
    }
}
check.dependsOn 'spotlessApply'

@SuppressWarnings('GroovyAssignabilityCheck')
def Task getPublishArtifactsTask(String buildVersion, Project subproject) {
    // generate a unique task name based on the directory name.
    return task("publishArtifact-$subproject.name" {
        apply plugin: 'maven-publish'
        publishing {
            repositories {
                publications {
                    "$subproject.name"(MavenPublication) {
                        from subproject.components.java

                        // use the subproject group and name with the assumption there are no identical subproject
                        // names, group names or subproject group/name combination.
                        groupId = "$subproject.group"
                        artifactId = "$subproject.name"
                        version = "$buildVersion"
                        repositories.add(rootProject.repositories.getByName('cloudrepo'))
                    }
                }
            }
        }
    })
}

allprojects {
    apply plugin: 'com.bmuschko.docker-remote-api'

    task copyDocker(type: Sync) {
        from "${project.projectDir}/Dockerfile"
        into "build/docker/"
    }
}

allprojects {
    apply plugin: 'base'

    // by default gradle uses directory as the project name. That works very well in a single project environment but
    // projects clobber each other in an environments with subprojects when projects are in directories named identically.
    def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
    group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
    project.archivesBaseName = "${project.group}-${project.name}"

    version = rootProject.ext.version
}

// Java projects common configurations
subprojects { subproj ->


    configurations {
        runtimeClasspath
    }


    // Common Docker Configuration:
    // If subprojects have the dockerImageName property configured in their gradle.properties file,
    // register:
    // 1) A copyGeneratedTar task to copy generated TARs. Subprojects that produce TARs can use this
    //    to copy the produced tar into the docker image.
    // 2) A buildDockerImage task to build the docker image.
    // 3) Make the docker image task depend on the default assemble task.
    if (subproj.hasProperty('dockerImageName')) {
        project.logger.lifecycle("configuring docker task for $subproj.name")

        // Although this task is defined for every subproject with the dockerImageName property,
        // It is not necessarily used for all subprojects. Non-TAR producing projects can ignore this.
        tasks.register("copyGeneratedTar", Copy) {
            dependsOn copyDocker
            dependsOn distTar
            from('build/distributions') {
                // Assume that tars are named <parent-project-name>-<child-project-name>-*.tar.
                // Because we only have a handle to the child project, and to keep things simple,
                // use a * regex to catch all prefixes.
                include "*$subproj.name-*.tar"
            }
            into 'build/docker/bin'
        }

        tasks.register("buildDockerImage", DockerBuildImage) {
            // This is currently only used for connectors.
            def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.4'

            def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
            def isArm64 = arch == "aarch64" || arch == "arm64"
            def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
            def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
            def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
            def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine'

            // Used by the platform -- Must be an Amazon Corretto-based image for build to work without modification to Dockerfile instructions
            def openjdkImage = System.getenv('JDK_IMAGE') ?: "airbyte/airbyte-base-java-image:1.0"

            platform = buildPlatform
            images.add("airbyte/$subproj.dockerImageName:$rootProject.ext.image_tag")
            buildArgs.put('JDK_VERSION', jdkVersion)
            buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
            buildArgs.put('ALPINE_IMAGE', alpineImage)
            buildArgs.put('NGINX_IMAGE', nginxImage)
            buildArgs.put('JDK_IMAGE', openjdkImage)
            buildArgs.put('VERSION', rootProject.ext.version)

        }

        tasks.named("assemble") {
            dependsOn buildDockerImage
        }
    }

    if (subproj.name == 'airbyte-webapp' || subproj.name == 'airbyte-webapp-e2e-tests') {
        return
    }

    apply plugin: 'java'
    apply plugin: 'jacoco'
    apply plugin: 'com.github.spotbugs'
    apply plugin: 'pmd'

    sourceCompatibility = JavaVersion.VERSION_17
    targetCompatibility = JavaVersion.VERSION_17

    repositories {
        mavenCentral()
        maven {
            url 'https://jitpack.io'
        }
        maven {
            url 'https://maven.twttr.com'
        }
    }

    pmd {
        consoleOutput = true
        ignoreFailures = false
        rulesMinimumPriority = 5
        ruleSets = []
        ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml'))
        toolVersion = '6.43.0'
    }

    jacoco {
        toolVersion = "0.8.7"
    }

    jacocoTestReport {
        dependsOn test
        reports {
            html.required = true
            xml.required = true
            csv.required = false
        }
    }

    jacocoTestCoverageVerification {
        violationRules {
            failOnViolation = false
            rule {
                element = 'CLASS'
                excludes = ['**/*Test*', '**/generated*']
                limit {
                    counter = 'BRANCH'
                    minimum = 0.8
                }
                limit {
                    counter = 'INSTRUCTION'
                    minimum = 0.8
                }
            }
        }
    }

    def integrationTagName = 'platform-integration'
    def slowIntegrationTagName = 'platform-slow-integration'
    // make tag accessible in submodules.
    ext {
        cloudStorageTestTagName = 'cloud-storage'
    }

    spotbugs {
        ignoreFailures = false
        effort = 'max'
        excludeFilter = rootProject.file('spotbugs-exclude-filter-file.xml')
        reportLevel = 'high'
        showProgress = false
        toolVersion = '4.6.0'
    }

    test {
        jacoco {
            enabled = true
            excludes = ['**/*Test*', '**/generated*']
        }
        useJUnitPlatform {
            excludeTags(integrationTagName, slowIntegrationTagName, cloudStorageTestTagName)
        }
        testLogging() {
            events "passed", "skipped", "failed"
            exceptionFormat 'full'
            // uncomment to get the full log output
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    task newIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags integrationTagName
        }
        testLogging() {
            events "passed", "failed", "started"
            exceptionFormat 'full'
            // uncomment to get the full log output
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    task slowIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags slowIntegrationTagName
        }
        testLogging() {
            events "passed", "failed", "started"
            exceptionFormat 'full'
            // uncomment to get the full log output
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    task allTests(type: Test) {
        useJUnitPlatform()
        testLogging() {
            events "passed", "failed", "started"
            exceptionFormat 'full'
            // uncomment to get the full log output
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    dependencies {
        if (subproj.name != 'airbyte-commons') {
            implementation project(':airbyte-commons')
        }

        implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0"))
        implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))


        // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
        implementation libs.bundles.jackson

        implementation libs.guava

        implementation libs.commons.io

        implementation libs.bundles.apache

        implementation libs.slf4j.api


        // SLF4J as a facade over Log4j2 required dependencies
        implementation libs.bundles.log4j

        // Bridges from other logging implementations to SLF4J
        implementation libs.bundles.slf4j

        // Dependencies for logging to cloud storage, as well as the various clients used to do so.
        implementation libs.appender.log4j2
        implementation libs.aws.java.sdk.s3
        implementation libs.google.cloud.storage

        implementation libs.s3

        // Lombok dependencies
        compileOnly libs.lombok
        annotationProcessor libs.lombok

        testCompileOnly libs.lombok
        testAnnotationProcessor libs.lombok

        testRuntimeOnly libs.junit.jupiter.engine
        testImplementation libs.bundles.junit
        testImplementation libs.assertj.core

        testImplementation libs.junit.pioneer

        // adds owasp plugin
        spotbugsPlugins libs.findsecbugs.plugin
        implementation libs.spotbugs.annotations
    }

    tasks.withType(Tar) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(Zip) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(SpotBugsTask) {
        // Reports can be found under each subproject in build/spotbugs/
        reports {
            xml.required = false
            html.required = true
        }
    }

    tasks.withType(Pmd) {
        exclude '**/generated/**'
        exclude '**/jooq/**'
    }

    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
    check.dependsOn 'jacocoTestCoverageVerification'
}

task('generate') {
    dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
    dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }
}

task('format') {
    dependsOn generate
    dependsOn spotlessApply
    dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) }
}

// add licenses for python projects.
subprojects {
    def pythonFormatTask = project.tasks.findByName('blackFormat')

    if (pythonFormatTask != null) {
        apply plugin: "com.github.hierynomus.license"
        license {
            header rootProject.file("LICENSE_SHORT")
        }
        task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = fileTree(dir: projectDir)
                    .include("**/*.py")
                    .exclude(".venv/**/*.py")
                    .exclude("**/airbyte_api_client/**/*.py")
                    .exclude("**/__init__.py")
            strictCheck = true
        }
        def licenseTask = project.tasks.findByName('licenseFormatPython')
        blackFormat.dependsOn licenseTask
        isortFormat.dependsOn licenseTask
        flakeCheck.dependsOn licenseTask

        def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles')
        if (generateFilesTask != null) {
            licenseTask.dependsOn generateFilesTask
        }
    }
    task listAllDependencies(type: DependencyReportTask) {}
}

task('generate-docker') {
    dependsOn(':airbyte-bootloader:assemble')
    dependsOn(':airbyte-workers:assemble')
    dependsOn(':airbyte-webapp:assemble')
    dependsOn(':airbyte-server:assemble')
    dependsOn(':airbyte-db:db-lib:assemble')
    dependsOn(':airbyte-config:init:assemble')
    dependsOn(':airbyte-temporal:assemble')
}

// produce reproducible archives
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
tasks.withType(AbstractArchiveTask) {
    preserveFileTimestamps = false
    reproducibleFileOrder = true
}

// definition for publishing
catalog {
    versionCatalog {
        from(files("deps.toml"))
    }
}

publishing {
    publications {
        // This block is present for dependency catalog publishing.
        maven(MavenPublication) {
            groupId = 'io.airbyte'
            artifactId = 'oss-catalog'

            from components.versionCatalog
            // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id.
            // e.g. the subproject :airbyte-example:example-models is imported at io.airbyte.airbyte-config:config-persistence:<version-number>.
        }
    }
    repositories.add(rootProject.repositories.getByName('cloudrepo'))
}