airbyte/airbyte-integrations/bases/base-normalization/build.gradle

import org.apache.tools.ant.taskdefs.condition.Os

plugins {
    id 'airbyte-docker'
    id 'airbyte-python'
}

airbytePython {
    moduleDirectory 'normalization'
}

dependencies {
    implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness')
}

// we need to access the sshtunneling script from airbyte-workers for ssh support
task copySshScript(type: Copy, dependsOn: [project(':airbyte-connector-test-harnesses:acceptance-test-harness').processResources]) {
    from "${project(':airbyte-connector-test-harnesses:acceptance-test-harness').buildDir}/resources/main"
    into "${buildDir}"
    include "sshtunneling.sh"
}

// make sure the copy task above worked (if it fails, it fails silently annoyingly)
task checkSshScriptCopy(type: Task, dependsOn: copySshScript) {
    doFirst {
        assert file("${buildDir}/sshtunneling.sh").exists() :
                "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers."
    }
}

airbyteDocker.dependsOn(checkSshScriptCopy)
assemble.dependsOn(checkSshScriptCopy)
test.dependsOn(checkSshScriptCopy)

integrationTest.dependsOn(build)


static def getDockerfile(String customConnector) {
    return "${customConnector}.Dockerfile"
}

static def getDockerImageName(String customConnector) {
    return "airbyte/normalization-${customConnector}"
}

static def getImageNameWithTag(String customConnector) {
    return "${getDockerImageName(customConnector)}:dev"
}


def buildAirbyteDocker(String customConnector) {
    // def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)]
    // As the base dbt image (https://hub.docker.com/r/fishtownanalytics/dbt/tags) we are using is only build for amd64, we need to use buildkit to force builds for your local environment
    // We are lucky that all the python code dbt uses is mutli-arch compatible

    def arch = 'linux/amd64'
    if (Os.isArch("aarch_64") || Os.isArch("aarch64")) {
        arch = 'linux/arm64'
    }

    def cmdArray = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.']
    // println("Building normalization container: " + cmdArray.join(" "))

    return {
        commandLine cmdArray
    }
}

task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('mssql')
    dependsOn assemble
}
task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('mysql')
    dependsOn assemble
}
task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('oracle')
    dependsOn assemble
}
task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('clickhouse')
    dependsOn assemble
}
task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('snowflake')
    dependsOn assemble
}
task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('redshift')
    dependsOn assemble
}
task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('tidb')
    dependsOn assemble
}
task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) {
    configure buildAirbyteDocker('duckdb')
    dependsOn assemble
}

airbyteDocker.dependsOn(airbyteDockerMSSql)
airbyteDocker.dependsOn(airbyteDockerMySql)
airbyteDocker.dependsOn(airbyteDockerOracle)
airbyteDocker.dependsOn(airbyteDockerClickhouse)
airbyteDocker.dependsOn(airbyteDockerSnowflake)
airbyteDocker.dependsOn(airbyteDockerRedshift)
airbyteDocker.dependsOn(airbyteDockerTiDB)
airbyteDocker.dependsOn(airbyteDockerDuckDB)

task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
    module = "pytest"
    command = "-s integration_tests"

    dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker'
    dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker'
}

// not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests
// and as such it depends on the docker images.
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker'
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker'

// DATs have some additional tests that exercise normalization code paths,
// so we want to run these in addition to the base-normalization integration tests.
// If you add more items here, make sure to also to have CI fetch their credentials.
// See git history for an example.
// TODO reenable these - they're causing flakiness in our test results, need to figure that out
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest")
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest")
// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest")

integrationTest.dependsOn("customIntegrationTestPython")
customIntegrationTests.dependsOn("customIntegrationTestPython")

// TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead
task('mypyCheck', type: PythonTask) {
    module = "mypy"
    command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"

    dependsOn 'blackFormat'
}
check.dependsOn mypyCheck