116 lines
3.9 KiB
Groovy
116 lines
3.9 KiB
Groovy
import java.nio.file.Paths
|
|
|
|
plugins {
|
|
id 'airbyte-docker'
|
|
id 'airbyte-python'
|
|
}
|
|
|
|
airbytePython {
|
|
moduleDirectory 'normalization'
|
|
}
|
|
|
|
dependencies {
|
|
implementation project(':airbyte-workers')
|
|
}
|
|
|
|
// we need to access the sshtunneling script from airbyte-workers for ssh support
|
|
task copySshScript(type: Copy, dependsOn: [project(':airbyte-workers').processResources]) {
|
|
from "${project(':airbyte-workers').buildDir}/resources/main"
|
|
into "${buildDir}"
|
|
include "sshtunneling.sh"
|
|
}
|
|
|
|
// make sure the copy task above worked (if it fails, it fails silently annoyingly)
|
|
task checkSshScriptCopy(type: Task, dependsOn: copySshScript) {
|
|
doFirst {
|
|
assert file("${buildDir}/sshtunneling.sh").exists() :
|
|
"Copy of sshtunneling.sh failed, check that it is present in airbyte-workers."
|
|
}
|
|
}
|
|
|
|
airbyteDocker.dependsOn(checkSshScriptCopy)
|
|
assemble.dependsOn(checkSshScriptCopy)
|
|
test.dependsOn(checkSshScriptCopy)
|
|
|
|
integrationTest.dependsOn(build)
|
|
|
|
|
|
static def getDockerfile(String customConnector) {
|
|
return "${customConnector}.Dockerfile"
|
|
}
|
|
|
|
static def getDockerImageName(String customConnector) {
|
|
return "airbyte/normalization-${customConnector}"
|
|
}
|
|
|
|
static def getImageNameWithTag(String customConnector) {
|
|
return "${getDockerImageName(customConnector)}:dev"
|
|
}
|
|
|
|
|
|
def buildAirbyteDocker(String customConnector) {
|
|
def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)]
|
|
return {
|
|
commandLine baseCommand
|
|
}
|
|
}
|
|
|
|
task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('mssql')
|
|
dependsOn assemble
|
|
}
|
|
task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('mysql')
|
|
dependsOn assemble
|
|
}
|
|
task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('oracle')
|
|
dependsOn assemble
|
|
}
|
|
task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('clickhouse')
|
|
dependsOn assemble
|
|
}
|
|
task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('snowflake')
|
|
dependsOn assemble
|
|
}
|
|
task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) {
|
|
configure buildAirbyteDocker('redshift')
|
|
dependsOn assemble
|
|
}
|
|
|
|
airbyteDocker.dependsOn(airbyteDockerMSSql)
|
|
airbyteDocker.dependsOn(airbyteDockerMySql)
|
|
airbyteDocker.dependsOn(airbyteDockerOracle)
|
|
airbyteDocker.dependsOn(airbyteDockerClickhouse)
|
|
airbyteDocker.dependsOn(airbyteDockerSnowflake)
|
|
airbyteDocker.dependsOn(airbyteDockerRedshift)
|
|
|
|
task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
|
|
module = "pytest"
|
|
command = "-s integration_tests"
|
|
|
|
dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
|
|
dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
|
|
}
|
|
|
|
integrationTest.dependsOn("customIntegrationTestPython")
|
|
customIntegrationTests.dependsOn("customIntegrationTestPython")
|
|
|
|
// TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead
|
|
task('mypyCheck', type: PythonTask) {
|
|
module = "mypy"
|
|
command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"
|
|
|
|
dependsOn 'blackFormat'
|
|
}
|
|
check.dependsOn mypyCheck
|