airbyte/build.gradle

import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage

buildscript {
    repositories {
        maven {
            url "https://plugins.gradle.org/m2/"
        }
    }
    dependencies {
        classpath 'com.bmuschko:gradle-docker-plugin:7.1.0'
    }
}

plugins {
    id 'base'
    id 'pmd'
    id 'com.diffplug.spotless' version '5.7.0'
    id 'com.github.hierynomus.license' version '0.16.1'
}

repositories {
    mavenCentral()
}

Properties env = new Properties()
rootProject.file('.env').withInputStream { env.load(it) }

if (!env.containsKey('VERSION')) {
    throw new Exception('Version not specified in .env file...')
}

def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
    /*
    In java, we don't have a second linter/styling tool other than spotless so it doesn't really
    matter if we write a newline or not for startComment/endComment.

    However, in python, we are using black that double-checks and reformats the code.
    Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
    big deal and would be reformatted (removed) because of black's specs.
    */
    def tmp = File.createTempFile('tmp', '.tmp')
    tmp.withWriter {
        def w = it
        if (startComment.length() > 0 || !isPython) {
            w.writeLine(startComment)
        }
        license.eachLine {
            w << lineComment
            w.writeLine(it)
        }
        if (endComment.length() > 0 || !isPython) {
            w.writeLine(endComment)
        }
        w.writeLine("")
        if (isPython) {
            w.writeLine("")
        }
    }
    return tmp
}

def createPythonLicenseWith = { license ->
    return createLicenseWith(license, '', '', '', true)
}

def createJavaLicenseWith = { license ->
    return createLicenseWith(license, '/*', ' */', ' * ', false)
}

// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable.
def createSpotlessTarget = { pattern ->
    def excludes = [
        '.gradle',
        'node_modules',
        '.eggs',
        '.mypy_cache',
        '.venv',
        '*.egg-info',
        'build',
        'dbt-project-template',
        'dbt-project-template-mssql',
        'dbt-project-template-mysql',
        'dbt-project-template-oracle',
        'dbt_test_config',
        'normalization_test_output',
        'tools',
        'secrets',
        'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
        'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file.
    ]

    if (System.getenv().containsKey("SUB_BUILD")) {
        excludes.add("airbyte-integrations/connectors")
    }

    return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" })
}

spotless {
    java {
        target createSpotlessTarget('**/*.java')

        importOrder()

        eclipse('4.16.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))

        licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
        removeUnusedImports()
        trimTrailingWhitespace()
    }
    groovyGradle {
        target createSpotlessTarget('**/*.gradle')
    }
    sql {
        target createSpotlessTarget('**/*.sql')

        dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
    }
    format 'styling', {
        target createSpotlessTarget(['**/*.yaml', '**/*.json'])

        prettier()
    }
}
check.dependsOn 'spotlessApply'

@SuppressWarnings('GroovyAssignabilityCheck')
def Task getDockerBuildTask(String artifactName, String projectDir) {
    return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
        def buildTag = System.getenv('VERSION') ?: 'dev'
        def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: 'linux/amd64'
        def alpineImage = System.getenv('ALPINE_IMAGE') ?: 'alpine:3.4'
        def postgresImage = System.getenv('POSTGRES_IMAGE') ?: 'postgres:13-alpine'
        def jdkVersion = System.getenv('JDK_VERSION') ?: '14.0.2'
        def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: 'amd64'

        inputDir = file("$projectDir/build/docker")
        platform = buildPlatform
        images.add("airbyte/$artifactName:$buildTag")
        buildArgs.put('JDK_VERSION', jdkVersion)
        buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
        buildArgs.put('ALPINE_IMAGE', alpineImage)
        buildArgs.put('POSTGRES_IMAGE', postgresImage)
    })
}

allprojects {
    apply plugin: 'com.bmuschko.docker-remote-api'

    task copyDocker(type: Copy) {
        delete "build/docker"

        from "${project.projectDir}/Dockerfile"
        into "build/docker/"
    }
}

allprojects {
    apply plugin: 'base'

    // by default gradle uses directory as the project name. That works very well in a single project environment but
    // projects clobber each other in an environments with subprojects when projects are in directories named identically.
    def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
    group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
    project.archivesBaseName = "${project.group}-${project.name}"

    version = env.VERSION
}

// Java projects common configurations
subprojects {

    if (project.name == 'airbyte-webapp' || project.name == 'airbyte-e2e-testing') {
        return
    }

    apply plugin: 'java'
    apply plugin: 'jacoco'

    sourceCompatibility = JavaVersion.VERSION_14
    targetCompatibility = JavaVersion.VERSION_14

    repositories {
        mavenCentral()
        maven {
            // TODO(Issue-4915): Remove this when upstream is merged in.
            url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
        }
    }

    pmd {
        consoleOutput = true

        rulesMinimumPriority = 5
        ruleSets = []
        ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml'))
    }

    def integrationTagName = 'platform-integration'
    def slowIntegrationTagName = 'platform-slow-integration'

    test {
        useJUnitPlatform {
            excludeTags(integrationTagName, slowIntegrationTagName)
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
            // showStandardStreams = true
        }
        finalizedBy jacocoTestReport
    }

    task newIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags integrationTagName
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    task slowIntegrationTests(type: Test) {
        useJUnitPlatform {
            includeTags slowIntegrationTagName
        }
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    task allTests(type: Test) {
        useJUnitPlatform()
        testLogging() {
            events 'failed'
            exceptionFormat 'full'
        }
        finalizedBy jacocoTestReport
    }

    dependencies {
        if (project.name != 'airbyte-commons') {
            implementation project(':airbyte-commons')
        }

        implementation(platform("com.fasterxml.jackson:jackson-bom:2.10.4"))
        implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))


        // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
        implementation 'com.fasterxml.jackson.core:jackson-databind'
        implementation 'com.fasterxml.jackson.core:jackson-annotations'
        implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
        implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'

        implementation 'com.google.guava:guava:30.1.1-jre'

        implementation 'commons-io:commons-io:2.7'

        implementation 'org.apache.commons:commons-compress:1.20'
        implementation 'org.apache.commons:commons-lang3:3.11'

        implementation 'org.slf4j:slf4j-api:1.7.30'


        // SLF4J as a facade over Log4j2 required dependencies
        implementation 'org.apache.logging.log4j:log4j-api:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-core:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.11.0'
        implementation 'org.apache.logging.log4j:log4j-web:2.14.1'

        // Bridges from other logging implementations to SLF4J
        implementation 'org.slf4j:jul-to-slf4j:1.7.30' // JUL bridge
        implementation 'org.slf4j:jcl-over-slf4j:1.7.30' //JCL bridge
        implementation 'org.slf4j:log4j-over-slf4j:1.7.30' // log4j1.2 bridge

        // Dependencies for logging to cloud storage, as well as the various clients used to do so.
        implementation 'com.therealvan:appender-log4j2:3.1.2-SNAPSHOT'
        implementation 'com.amazonaws:aws-java-sdk-s3:1.12.6'
        implementation 'com.google.cloud:google-cloud-storage:1.115.0'

        implementation 'software.amazon.awssdk:s3:2.16.84'

        // Lombok dependencies
        compileOnly 'org.projectlombok:lombok:1.18.22'
        annotationProcessor 'org.projectlombok:lombok:1.18.22'

        testCompileOnly 'org.projectlombok:lombok:1.18.22'
        testAnnotationProcessor 'org.projectlombok:lombok:1.18.22'

        testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.2'
        testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2'
        testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
        testImplementation 'org.mockito:mockito-junit-jupiter:3.12.4'
        testImplementation 'org.assertj:assertj-core:3.21.0'

    }

    tasks.withType(Tar) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    tasks.withType(Zip) {
        duplicatesStrategy DuplicatesStrategy.INCLUDE
    }

    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
}

task('generate') {
    dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
    dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }
}

task('format') {
    dependsOn generate
    dependsOn spotlessApply
    dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) }
}

// add licenses for python projects.
subprojects {
    def pythonFormatTask = project.tasks.findByName('blackFormat')

    if (pythonFormatTask != null) {
        apply plugin: "com.github.hierynomus.license"
        task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) {
            header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
            source = fileTree(dir: projectDir)
                    .include("**/*.py")
                    .exclude(".venv/**/*.py")
                    .exclude("**/__init__.py")
            strictCheck = true
        }
        def licenseTask = project.tasks.findByName('licenseFormatPython')
        blackFormat.dependsOn licenseTask
        isortFormat.dependsOn licenseTask
        flakeCheck.dependsOn licenseTask

        def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles')
        if (generateFilesTask != null) {
            licenseTask.dependsOn generateFilesTask
        }
    }
}

task('generate-docker') {
    dependsOn(':airbyte-workers:assemble')
    dependsOn(':airbyte-webapp:assemble')
    dependsOn(':airbyte-server:assemble')
    dependsOn(':airbyte-scheduler:app:assemble')
    dependsOn(':airbyte-migration:assemble')
    dependsOn(':airbyte-db:lib:assemble')
    dependsOn(':airbyte-config:init:assemble')
    dependsOn(':airbyte-temporal:assemble')
}

// produce reproducible archives
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
tasks.withType(AbstractArchiveTask) {
    preserveFileTimestamps = false
    reproducibleFileOrder = true
}

// TODO: Davin will turn this back on.
//// Configure what subprojects to publish.
//String[] toPublish = [
//        ':airbyte-analytics',
//        ':airbyte-api',
//        ':airbyte-commons',
//        ':airbyte-commons-docker',
//        ':airbyte-config:init',
//        ':airbyte-config:models',
//        ':airbyte-config:persistence',
//        ':airbyte-db:lib',
//        ':airbyte-json-validation',
//        ':airbyte-migration',
//        ':airbyte-notification',
//        ':airbyte-protocol:models',
//        ':airbyte-scheduler:client',
//        ':airbyte-scheduler:models',
//        ':airbyte-scheduler:persistence',
//        ':airbyte-server',
//        ':airbyte-workers'
//]
//configure(subprojects.findAll { toPublish.contains(it.getPath()) }) {
//    apply plugin: 'maven-publish'
//
//    publishing {
//        repositories {
//            publications {
//                // This block is present so Gradle knows to publish a Maven jar.
//                maven(MavenPublication) {
//                    from components.java
//                    // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id.
//                    // e.g. the subproject :airbyte-scheduler:models is imported at io.airbyte.airbyte-config:persistence:<version-number>.
//                }
//            }
//
//            maven {
//                credentials {
//                    name 'cloudrepo'
//                    username System.getenv('CLOUDREPO_USER')
//                    password System.getenv('CLOUDREPO_PASSWORD')
//                }
//                url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars'
//            }
//
//            mavenLocal()
//        }
//    }
//}