gradle: fix airbyteDocker task inputs (#30187)
This commit is contained in:
Marius Posta
@@ -11,7 +11,7 @@ tasks.register('generateComponentManifestClassFiles', Exec) {
|
||||
environment 'ROOT_DIR', rootDir.absolutePath
|
||||
commandLine 'bin/generate-component-manifest-files.sh'
|
||||
}.configure {
|
||||
dependsOn project(':tools:code-generator').tasks.named('airbyteDocker')
|
||||
dependsOn project(':tools:code-generator').tasks.named('assemble')
|
||||
}
|
||||
|
||||
tasks.register('validateSourceYamlManifest', Exec) {
|
||||
|
||||
@@ -28,3 +28,7 @@ dependencies {
|
||||
testImplementation 'commons-lang:commons-lang:2.6'
|
||||
implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2'
|
||||
}
|
||||
|
||||
tasks.named('airbyteDocker').configure {
|
||||
dependsOn project(':airbyte-integrations:bases:base').tasks.named('assemble')
|
||||
}
|
||||
@@ -37,9 +37,6 @@ checkSshScriptCopy.configure {
|
||||
tasks.named('airbyteDocker').configure {
|
||||
dependsOn checkSshScriptCopy
|
||||
}
|
||||
tasks.named('assemble').configure {
|
||||
dependsOn checkSshScriptCopy
|
||||
}
|
||||
tasks.named('check').configure {
|
||||
dependsOn checkSshScriptCopy
|
||||
}
|
||||
@@ -50,7 +47,7 @@ def customIntegrationTestPython = tasks.register('customIntegrationTestPython',
|
||||
}
|
||||
customIntegrationTestPython.configure {
|
||||
dependsOn tasks.named('installTestReqs')
|
||||
dependsOn tasks.named('airbyteDocker')
|
||||
dependsOn tasks.named('assemble')
|
||||
}
|
||||
|
||||
static def getDockerfile(String customConnector) {
|
||||
@@ -97,16 +94,15 @@ def buildAirbyteDocker(String customConnector) {
|
||||
}
|
||||
task.configure {
|
||||
dependsOn checkSshScriptCopy
|
||||
dependsOn tasks.named('assemble')
|
||||
}
|
||||
tasks.named('airbyteDocker').configure {
|
||||
tasks.named('assemble').configure {
|
||||
dependsOn task
|
||||
}
|
||||
}
|
||||
|
||||
def customIntegrationTestsCoverage = tasks.named('_customIntegrationTestsCoverage')
|
||||
customIntegrationTestsCoverage.configure {
|
||||
dependsOn tasks.named('airbyteDocker')
|
||||
dependsOn tasks.named('assemble')
|
||||
}
|
||||
|
||||
[
|
||||
@@ -123,12 +119,12 @@ customIntegrationTestsCoverage.configure {
|
||||
].each {destinationName ->
|
||||
def destinationProject = project(":airbyte-integrations:connectors:destination-$destinationName")
|
||||
customIntegrationTestPython.configure {
|
||||
dependsOn destinationProject.tasks.named('airbyteDocker')
|
||||
dependsOn destinationProject.tasks.named('assemble')
|
||||
}
|
||||
// Not really sure what this task does differently from customIntegrationTestPython,
|
||||
// but it seems to also run integration tests and as such it depends on the docker images.
|
||||
customIntegrationTestsCoverage.configure {
|
||||
dependsOn destinationProject.tasks.named('airbyteDocker')
|
||||
dependsOn destinationProject.tasks.named('assemble')
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,4 +146,4 @@ def mypyCheck = tasks.register('mypyCheck', PythonTask) {
|
||||
module = "mypy"
|
||||
command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"
|
||||
}
|
||||
tasks.named('airbytePythonChecks').configure { dependsOn mypyCheck }
|
||||
tasks.named('check').configure { dependsOn mypyCheck }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
!entrypoint.sh
|
||||
|
||||
@@ -76,10 +76,6 @@ def generateSourceTestDocs = tasks.register('generateSourceTestDocs', Javadoc) {
|
||||
outputs.upToDateWhen { false }
|
||||
}
|
||||
|
||||
tasks.named('build').configure {
|
||||
dependsOn generateSourceTestDocs
|
||||
}
|
||||
|
||||
application {
|
||||
mainClass = 'io.airbyte.integrations.standardtest.source.PythonSourceAcceptanceTest'
|
||||
}
|
||||
|
||||
@@ -1,19 +1,3 @@
|
||||
plugins {
|
||||
id "base"
|
||||
id "com.github.node-gradle.node" version "3.5.1"
|
||||
}
|
||||
|
||||
def nodeVersion = System.getenv('NODE_VERSION') ?: '16.13.0'
|
||||
|
||||
node {
|
||||
download = true
|
||||
version = nodeVersion
|
||||
}
|
||||
|
||||
tasks.named('assemble').configure {
|
||||
dependsOn tasks.named('npmInstall')
|
||||
}
|
||||
|
||||
def generateScaffolds = tasks.register('generateScaffolds')
|
||||
|
||||
def addScaffoldTemplateTask(name, packageName, outputDirName, scaffoldParams=[]) {
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
!base.sh
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
!base.sh
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -63,8 +63,8 @@ dependencies {
|
||||
}
|
||||
|
||||
tasks.named('airbyteDocker').configure {
|
||||
// this is really inefficent (because base-normalization:airbyteDocker builds 9 docker images)
|
||||
// this is really inefficent (because base-normalization:assemble builds 9 docker images)
|
||||
// but it's also just simple to implement.
|
||||
// this also goes away once airbyte-ci becomes a reality.
|
||||
dependsOn project(':airbyte-integrations:bases:base-normalization').tasks.named('airbyteDocker')
|
||||
dependsOn project(':airbyte-integrations:bases:base-normalization').tasks.named('assemble')
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!build
|
||||
!build/distributions
|
||||
|
||||
@@ -140,6 +140,8 @@ allprojects {
|
||||
|
||||
tasks.withType(Zip).configureEach {
|
||||
duplicatesStrategy DuplicatesStrategy.INCLUDE
|
||||
// Disabling distZip causes the build to break for some reason, so: instead of disabling it, make it fast.
|
||||
entryCompression ZipEntryCompression.STORED
|
||||
}
|
||||
}
|
||||
|
||||
@@ -459,6 +461,9 @@ tasks.named('spotlessStyling').configure {
|
||||
dependsOn tasks.named('nodeSetup')
|
||||
dependsOn tasks.named('npmSetup')
|
||||
}
|
||||
tasks.matching { it.name =~ /spotless.*Check/ }.configureEach {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
// python is required by the root project to apply python formatters like isort or black.
|
||||
python {
|
||||
|
||||
@@ -47,12 +47,10 @@ class AirbyteConnectorAcceptanceTestPlugin implements Plugin<Project> {
|
||||
outputs.upToDateWhen { false }
|
||||
}
|
||||
connectorAcceptanceTest.configure {
|
||||
dependsOn project.tasks.named('build')
|
||||
dependsOn project.tasks.named('airbyteDocker')
|
||||
dependsOn project.tasks.named('assemble')
|
||||
if (project.connectorAcceptanceTestVersion == 'dev') {
|
||||
dependsOn project(':airbyte-integrations:bases:connector-acceptance-test').tasks.named('airbyteDocker')
|
||||
dependsOn project(':airbyte-integrations:bases:connector-acceptance-test').tasks.named('assemble')
|
||||
}
|
||||
dependsOn project.tasks.matching { it.name == 'airbyteDockerTest' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,85 +1,69 @@
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.Plugin
|
||||
import org.gradle.api.Project
|
||||
import org.gradle.api.file.ConfigurableFileTree
|
||||
import org.gradle.api.file.FileCollection
|
||||
import org.gradle.api.tasks.CacheableTask
|
||||
import org.gradle.api.tasks.Input
|
||||
import org.gradle.api.tasks.InputDirectory
|
||||
import org.gradle.api.tasks.InputFile
|
||||
import org.gradle.api.tasks.InputFiles
|
||||
import org.gradle.api.tasks.Internal
|
||||
import org.gradle.api.tasks.OutputFile
|
||||
import org.gradle.api.tasks.PathSensitive
|
||||
import org.gradle.api.tasks.PathSensitivity
|
||||
import org.gradle.api.tasks.TaskAction
|
||||
import org.slf4j.Logger
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.security.MessageDigest
|
||||
import java.nio.file.Paths
|
||||
|
||||
@CacheableTask
|
||||
abstract class AirbyteDockerTask extends DefaultTask {
|
||||
@Internal
|
||||
abstract File rootDir
|
||||
abstract class AirbyteDockerBaseTask extends DefaultTask {
|
||||
|
||||
@InputFiles
|
||||
@PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract FileCollection projectFiles
|
||||
FileCollection filesInDockerImage
|
||||
|
||||
@Input
|
||||
abstract Set<String> baseImageHashes
|
||||
Set<String> baseImageHashes
|
||||
|
||||
@InputDirectory
|
||||
@PathSensitive(PathSensitivity.RELATIVE)
|
||||
abstract File projectDir
|
||||
|
||||
@Input
|
||||
String dockerfileName
|
||||
|
||||
@Input
|
||||
boolean followSymlinks = false
|
||||
@InputFile
|
||||
File dockerFile
|
||||
|
||||
@OutputFile
|
||||
abstract File idFileOutput
|
||||
File idFileOutput
|
||||
}
|
||||
|
||||
def buildDockerfile(String scriptPath, String fileName) {
|
||||
if (project.file(fileName).exists()) {
|
||||
def tag = DockerHelpers.getDevTaggedImage(projectDir, dockerfileName)
|
||||
abstract class AirbyteDockerTask extends AirbyteDockerBaseTask {
|
||||
|
||||
def arch = System.getProperty("os.arch").toLowerCase()
|
||||
def isArm64 = arch == "aarch64" || arch == "arm64"
|
||||
def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'amd64'
|
||||
|
||||
project.exec {
|
||||
commandLine scriptPath, rootDir.absolutePath, projectDir.absolutePath, dockerfileName, tag, idFileOutput.absolutePath, followSymlinks, buildPlatform
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def buildDockerfileWithLocalCdk(String scriptPath, String fileName) {
|
||||
if (project.file(fileName).exists()) {
|
||||
def tag = DockerHelpers.getDevTaggedImage(projectDir, dockerfileName)
|
||||
project.exec {
|
||||
environment "CONNECTOR_TAG", tag
|
||||
environment "CONNECTOR_NAME", project.findProperty('connectorAcceptanceTest.connectorName')
|
||||
commandLine scriptPath
|
||||
}
|
||||
}
|
||||
}
|
||||
@InputFile
|
||||
File buildScript = project.rootProject.file('tools/bin/build_image.sh')
|
||||
|
||||
@TaskAction
|
||||
def dockerTask() {
|
||||
if (
|
||||
project.hasProperty('connectorAcceptanceTest.useLocalCdk') &&
|
||||
project.properties["connectorAcceptanceTest.useLocalCdk"] &&
|
||||
project.parent.project.name.equals("connectors")
|
||||
) {
|
||||
def scriptPath = Paths.get(rootDir.absolutePath, 'airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh').toString()
|
||||
buildDockerfileWithLocalCdk(scriptPath, dockerfileName)
|
||||
project.exec {
|
||||
commandLine(
|
||||
buildScript.absolutePath,
|
||||
project.rootDir.absolutePath,
|
||||
project.projectDir.absolutePath,
|
||||
dockerFile.name,
|
||||
DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile.name),
|
||||
idFileOutput.absolutePath,
|
||||
)
|
||||
}
|
||||
else {
|
||||
def scriptPath = Paths.get(rootDir.absolutePath, 'tools/bin/build_image.sh').toString()
|
||||
buildDockerfile(scriptPath, dockerfileName)
|
||||
}
|
||||
}
|
||||
|
||||
abstract class AirbyteDockerWithLocalCDKLegacyTask extends AirbyteDockerBaseTask {
|
||||
|
||||
@InputFile
|
||||
File buildScript = project.rootProject.file('airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh')
|
||||
|
||||
@TaskAction
|
||||
def dockerTask() {
|
||||
project.exec {
|
||||
environment "CONNECTOR_TAG", DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile.name)
|
||||
environment "CONNECTOR_NAME", project.findProperty('connectorAcceptanceTest.connectorName')
|
||||
commandLine buildScript.absolutePath
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,9 +135,9 @@ class AirbyteDockerPlugin implements Plugin<Project> {
|
||||
return "$stdout".toString().trim()
|
||||
}
|
||||
|
||||
static boolean isUpToDate(Logger logger, File idFileOutput, Project project, String dockerFile, Path dockerPath) {
|
||||
static boolean isUpToDate(Logger logger, File idFileOutput, Project project, File dockerFile) {
|
||||
if (idFileOutput.exists()) {
|
||||
def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile)
|
||||
def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile.name)
|
||||
logger.debug "taggedImage " + taggedImage
|
||||
|
||||
def current = getImageHash(project, taggedImage)
|
||||
@@ -161,7 +145,7 @@ class AirbyteDockerPlugin implements Plugin<Project> {
|
||||
def stored = (String) project.rootProject.imageToHash.get(taggedImage)
|
||||
logger.debug "stored " + stored
|
||||
|
||||
def notUpToDate = new ArrayList<String>(getBaseTaggedImages(dockerPath.toFile())).any { baseImage ->
|
||||
def notUpToDate = new ArrayList<String>(getBaseTaggedImages(dockerFile)).any { baseImage ->
|
||||
logger.debug "checking base image " + baseImage
|
||||
def storedBase = (String) project.rootProject.imageToHash.get(resolveEnvironmentVariables(project, baseImage))
|
||||
def currentBase = getImageHash(project, baseImage)
|
||||
@@ -193,36 +177,58 @@ class AirbyteDockerPlugin implements Plugin<Project> {
|
||||
}
|
||||
}
|
||||
|
||||
static def createTask(Project project, String taskName, String dockerFile) {
|
||||
if (project.file(dockerFile).exists()) {
|
||||
def filteredProjectFiles = project.fileTree(project.projectDir).filter {
|
||||
static FileCollection filteredProjectFiles(Project project) {
|
||||
ConfigurableFileTree files = project.fileTree(project.projectDir).asFileTree
|
||||
def dockerignore = project.file('.dockerignore')
|
||||
if (!dockerignore.exists()) {
|
||||
return files.filter {
|
||||
file -> !file.toString().contains(".venv")
|
||||
}
|
||||
}
|
||||
for (def rule : dockerignore.readLines()) {
|
||||
if (rule.startsWith("#")) {
|
||||
continue
|
||||
}
|
||||
rule = rule.trim()
|
||||
files = (rule.startsWith("!") ? files.include(rule.substring(1)) : files.exclude(rule)) as ConfigurableFileTree
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
def airbyteDockerTask = project.tasks.register(taskName, AirbyteDockerTask) {
|
||||
def dockerPath = Paths.get(project.projectDir.absolutePath, dockerFile)
|
||||
def hash = MessageDigest.getInstance("MD5").digest(dockerPath.getBytes()).encodeHex().toString()
|
||||
dockerfileName = dockerFile
|
||||
rootDir = project.rootProject.rootDir
|
||||
projectDir = project.projectDir
|
||||
projectFiles = filteredProjectFiles
|
||||
idFileOutput = project.file(Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', hash).toString())
|
||||
baseImageHashes = getBaseImageHashes(project.rootProject.imageToHash, dockerPath.toFile())
|
||||
|
||||
outputs.upToDateWhen {
|
||||
return isUpToDate(logger, idFileOutput, project, dockerFile, dockerPath)
|
||||
}
|
||||
}
|
||||
airbyteDockerTask.configure {
|
||||
dependsOn project.tasks.named('assemble')
|
||||
}
|
||||
project.tasks.named('build').configure {
|
||||
dependsOn airbyteDockerTask
|
||||
}
|
||||
} else {
|
||||
static def createTask(Project project, String taskName, String dockerFileName) {
|
||||
def dockerFile = project.file(dockerFileName)
|
||||
if (!dockerFile.exists()) {
|
||||
project.tasks.register(taskName) {
|
||||
logger.info "Skipping ${taskName} because ${dockerFile} does not exist."
|
||||
}
|
||||
return
|
||||
}
|
||||
Class<? extends AirbyteDockerBaseTask> taskClass = AirbyteDockerTask
|
||||
if (project.hasProperty('connectorAcceptanceTest.useLocalCdk') && project.parent.project.name == "connectors") {
|
||||
taskClass = AirbyteDockerWithLocalCDKLegacyTask
|
||||
}
|
||||
def airbyteDockerTask = project.tasks.register(taskName, taskClass) {
|
||||
def dockerfilePathHash = MessageDigest.getInstance("MD5")
|
||||
.digest(dockerFile.absolutePath.getBytes())
|
||||
.encodeHex()
|
||||
.toString()
|
||||
|
||||
it.filesInDockerImage = filteredProjectFiles(project)
|
||||
it.dockerFile = dockerFile
|
||||
it.baseImageHashes = getBaseImageHashes(project.rootProject.imageToHash, dockerFile)
|
||||
it.idFileOutput = Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', dockerfilePathHash).toFile()
|
||||
|
||||
it.outputs.upToDateWhen {
|
||||
return isUpToDate(logger, it.idFileOutput, project, dockerFile)
|
||||
}
|
||||
}
|
||||
airbyteDockerTask.configure {
|
||||
dependsOn project.tasks.matching { it.name == 'distTar' }
|
||||
dependsOn project.tasks.named('generate')
|
||||
}
|
||||
project.tasks.named('assemble').configure {
|
||||
dependsOn airbyteDockerTask
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ class AirbyteIntegrationTestJavaPlugin implements Plugin<Project> {
|
||||
}
|
||||
integrationTestJava.configure {
|
||||
mustRunAfter project.tasks.named('test')
|
||||
dependsOn project.tasks.matching { it.name == 'airbyteDocker' }
|
||||
dependsOn project.tasks.matching { it.name == 'assemble' }
|
||||
dependsOn project.tasks.matching { it.name == 'spotbugsMain' }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ class AirbytePerformanceTestJavaPlugin implements Plugin<Project> {
|
||||
}
|
||||
performanceTestJava.configure {
|
||||
mustRunAfter project.tasks.named('test')
|
||||
dependsOn project.tasks.matching { it.name == 'airbyteDocker' }
|
||||
dependsOn project.tasks.matching { it.name == 'assemble' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,8 +196,7 @@ class AirbytePythonPlugin implements Plugin<Project> {
|
||||
dependsOn installLocalReqs
|
||||
}
|
||||
|
||||
def airbytePythonChecks = project.tasks.register('airbytePythonChecks') {}
|
||||
airbytePythonChecks.configure {
|
||||
project.tasks.named('check').configure {
|
||||
dependsOn installReqs
|
||||
dependsOn flakeCheck
|
||||
}
|
||||
@@ -208,15 +207,11 @@ class AirbytePythonPlugin implements Plugin<Project> {
|
||||
command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('pyproject.toml').absolutePath}"
|
||||
}
|
||||
|
||||
airbytePythonChecks.configure {
|
||||
project.tasks.named('check').configure {
|
||||
dependsOn mypyCheck
|
||||
}
|
||||
}
|
||||
|
||||
project.tasks.named('assemble').configure {
|
||||
dependsOn airbytePythonChecks
|
||||
}
|
||||
|
||||
def installTestReqs = project.tasks.register('installTestReqs', PythonTask) {
|
||||
module = "pip"
|
||||
command = "install .[tests]"
|
||||
@@ -227,11 +222,8 @@ class AirbytePythonPlugin implements Plugin<Project> {
|
||||
dependsOn installReqs
|
||||
}
|
||||
|
||||
|
||||
|
||||
Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'unitTest', installTestReqs)
|
||||
project.tasks.named('check').configure {
|
||||
dependsOn airbytePythonChecks
|
||||
dependsOn project.tasks.matching { it.name == 'unitTest' }
|
||||
}
|
||||
|
||||
|
||||
@@ -65,10 +65,8 @@ class AirbyteStandardSourceTestFilePlugin implements Plugin<Project> {
|
||||
outputs.upToDateWhen { false }
|
||||
}
|
||||
standardSourceTestFile.configure {
|
||||
dependsOn project.project(':airbyte-integrations:bases:base-standard-source-test-file').tasks.named('airbyteDocker')
|
||||
dependsOn project.tasks.named('build')
|
||||
dependsOn project.tasks.matching { it.name == 'airbyteDocker' }
|
||||
dependsOn project.tasks.matching { it.name == 'airbyteDockerTest' }
|
||||
dependsOn project.project(':airbyte-integrations:bases:base-standard-source-test-file').tasks.named('assemble')
|
||||
dependsOn project.tasks.named('assemble')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
4
gradle/wrapper/gradle-wrapper.properties
vendored
4
gradle/wrapper/gradle-wrapper.properties
vendored
@@ -1,6 +1,6 @@
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
|
||||
networkTimeout=10000
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
zipStorePath=wrapper/dists
|
||||
|
||||
@@ -7,7 +7,6 @@ PROJECT_DIR="$2"
|
||||
DOCKERFILE="$3"
|
||||
TAGGED_IMAGE="$4"
|
||||
ID_FILE="$5"
|
||||
FOLLOW_SYMLINKS="$6"
|
||||
DOCKER_BUILD_ARCH="${DOCKER_BUILD_ARCH:-amd64}"
|
||||
# https://docs.docker.com/develop/develop-images/build_enhancements/
|
||||
export DOCKER_BUILDKIT=1
|
||||
@@ -18,34 +17,15 @@ assert_root
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
function validate_dockerignore() {
|
||||
excludes_all=$(grep -w '^\*$' .dockerignore || true)
|
||||
excludes_except=$(grep -w '^!.*' .dockerignore || true)
|
||||
if [ -n "$excludes_all" ] || [ -n "$excludes_except" ]; then
|
||||
error "Cannot include exclusion exceptions when following symlinks. Please use an exclude pattern that doesn't use exclude-all (e.g: *) or exclude-except (e.g: !/some/pattern)"
|
||||
fi
|
||||
}
|
||||
|
||||
args=(
|
||||
-f "$DOCKERFILE"
|
||||
-t "$TAGGED_IMAGE"
|
||||
--iidfile "$ID_FILE"
|
||||
)
|
||||
|
||||
if [ "$FOLLOW_SYMLINKS" == "true" ]; then
|
||||
exclusions=()
|
||||
if [ -f ".dockerignore" ]; then
|
||||
validate_dockerignore
|
||||
exclusions+=(--exclude-from .dockerignore)
|
||||
fi
|
||||
# Docker does not follow symlinks in the build context. So we create a tar of the directory, following symlinks, and provide the archive to Docker
|
||||
# to use as the build context
|
||||
tar cL "${exclusions[@]}" . | docker build - "${args[@]}"
|
||||
JDK_VERSION="${JDK_VERSION:-17.0.4}"
|
||||
if [[ -z "${DOCKER_BUILD_PLATFORM}" ]]; then
|
||||
docker build --build-arg JDK_VERSION="$JDK_VERSION" --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" . "${args[@]}"
|
||||
else
|
||||
JDK_VERSION="${JDK_VERSION:-17.0.4}"
|
||||
if [[ -z "${DOCKER_BUILD_PLATFORM}" ]]; then
|
||||
docker build --build-arg JDK_VERSION="$JDK_VERSION" --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" . "${args[@]}"
|
||||
else
|
||||
docker build --build-arg JDK_VERSION="$JDK_VERSION" --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" --platform="$DOCKER_BUILD_PLATFORM" . "${args[@]}"
|
||||
fi
|
||||
docker build --build-arg JDK_VERSION="$JDK_VERSION" --build-arg DOCKER_BUILD_ARCH="$DOCKER_BUILD_ARCH" --platform="$DOCKER_BUILD_PLATFORM" . "${args[@]}"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user