chore: non-dagger builds. (#59716)

Introduce the airbyte gradle docker plugin to build JVM connectors. Add logic to allow us to slowly migrate from one build set up to the other. The majority of the work is taking in the metadata.yaml file, and converting that to buildArgs to be injected into the airbyte gradle docker plugin. In particular, we extract the base image from the metadata file. We also add the connector name by referencing the module. This outputs a file buildArgs.properties into the build folder. This is done by the DockerGenerateConnectorBuildArgs task. The AirbyteDockerConventionPlugin then takes in the generated buildArgs.properties file and feeds it into the Platform Docker plugin. It also feeds in the top-level Dockerfile, so the Docker plugin can copy it into the build folder. I tried to migrate this to Kotlin, however the repo needs to be upgraded to the same Kotlin and KSP version as the gradle plugin repos for this to be worth while. Otherwise we end up writing kotlin plugins with reflection, which is worse than the current groovy plugins.
2025-12-19 18:14:56 -05:00 · 2025-05-08 17:04:22 -07:00
parent 02968945d0
commit 18e6370fc1
6 changed files with 186 additions and 7 deletions
--- a/airbyte-integrations/connectors/build.gradle
+++ b/airbyte-integrations/connectors/build.gradle
@@ -1,6 +1,7 @@
 plugins {
    id 'base'
    id 'ru.vyarus.use-python' version '2.3.0'
+    id "io.airbyte.gradle.docker" version "0.63.0" apply false
    id 'com.github.eirnym.js2p' version "1.0" apply false
 }

@@ -81,8 +82,8 @@ allprojects {
        return task
    }

-    // Build connector image as part of 'assemble' task.
-    // This is required for local 'integrationTest' execution.
+    // This allows us to slowly migrate connectors between dagger and the new plugins.
+    // This can be deleted once we no longer use Dagger.
    def buildConnectorImage = airbyteCIConnectorsTask(
            'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar')
    buildConnectorImage.configure {
@@ -91,10 +92,15 @@ allprojects {
        // Ensure that all files exist beforehand.
        dependsOn tasks.matching { it.name == 'generate' }
    }
+
+    afterEvaluate { Project p ->
+        if (!plugins.hasPlugin('airbyte-connector-docker-convention')) {
            tasks.named('assemble').configure {
                // We may revisit the dependency on assemble but the dependency should always be on a base task.
                dependsOn buildConnectorImage
            }
+        }
+    }

    // Convenience tasks for local airbyte-ci execution.
    airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build')
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -3,10 +3,15 @@ plugins {
 }

 repositories {
-    // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.
+    // Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.
    gradlePluginPortal()
 }

+dependencies {
+    // Note this dependency version needs to line up with the generateJsonSchema2Pojo plugin used in the project.
+    implementation 'org.yaml:snakeyaml:1.33'
+}
+
 tasks.withType(Jar).configureEach {
    duplicatesStrategy DuplicatesStrategy.INCLUDE
 }
--- a/buildSrc/src/main/groovy/airbyte-connector-docker-convention.gradle
+++ b/buildSrc/src/main/groovy/airbyte-connector-docker-convention.gradle
@@ -0,0 +1,85 @@
+import io.airbyte.gradle.DockerGenerateConnectorBuildArgs
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.file.RegularFile
+import org.gradle.api.plugins.ExtensionAware
+import org.gradle.api.provider.Provider
+
+class AirbyteConnectorDockerConventionPlugin implements Plugin<Project> {
+    private static final String SHARED_DOCKERFILE = 'docker-images/Dockerfile.java-connector-non-airbyte-ci'
+
+    @Override
+    void apply(Project project) {
+        /*
+            We deliberately avoid importing classes from the published Airbyte Docker
+            plugin. That keeps buildSrc free of a compile‑time dependency on a specific
+            plugin version and lets this convention plugin compile before external
+            plugins are resolved.
+         */
+
+        project.plugins.withId('io.airbyte.gradle.docker') {
+            def dockerExt = ((project.extensions.findByName('airbyte') as ExtensionAware)
+                    ?.extensions?.findByName('docker'))
+
+            if (!dockerExt) {
+                project.logger.warn('airbyte.docker extension not found; skipping convention plugin.')
+                return
+            }
+
+            /* ------------------------------------------------------------
+             * 1. imageName default = project.name (if not already set)
+             * ------------------------------------------------------------ */
+            if (!dockerExt.imageName.present) {
+                dockerExt.imageName.convention(project.name)
+            }
+
+            /* ------------------------------------------------------------
+             * 2. Use shared Dockerfile if not overridden. This allows
+             *    users to specify the Dockerfiles per connector.
+             *    The core plugin’s dockerCopyDockerfile task will copy
+             *    it into build/airbyte/docker/Dockerfile automatically.
+             * ------------------------------------------------------------ */
+            if (!dockerExt.dockerfile.present) {
+                dockerExt.dockerfile.set(project.rootProject.file(SHARED_DOCKERFILE))
+            }
+
+            /* ------------------------------------------------------------
+             * 3. Task: generate buildArgs.properties from metadata.yaml
+             * ------------------------------------------------------------ */
+            def genProps = project.tasks.register(
+                    'generateConnectorDockerBuildArgs',
+                    DockerGenerateConnectorBuildArgs
+            ) {
+                metadata.set(project.layout.projectDirectory.file('metadata.yaml'))
+                output.set(project.layout.buildDirectory.file('docker/buildArgs.properties'))
+            }
+
+            /* ------------------------------------------------------------
+             * 4. Lazy provider: read and inject the file generated by the
+             *    DockerGenerateConnectorBuildArgs task as buildArgs
+             * ------------------------------------------------------------ */
+            def propsMapProvider = project.providers
+                    .fileContents(genProps.flatMap { it.output } as Provider<RegularFile>)
+                    .asText
+                    .map { txt ->
+                        txt.readLines()
+                                .findAll { it.contains('=') }
+                                .collectEntries { ln ->
+                                    def (k, v) = ln.split('=', 2)
+                                    [(k.trim()): v.trim()]
+                                }
+                    }
+            dockerExt.buildArgs.putAll(propsMapProvider)
+
+            /* ------------------------------------------------------------
+             * 5. Make dockerBuildx depend on the generator, and register
+             *    the properties file as an input (incremental build‑cache)
+             * ------------------------------------------------------------ */
+            project.tasks.named('dockerBuildx').configure { Task t ->
+                t.dependsOn(genProps)
+                t.inputs.file(genProps.flatMap { it.output })
+            }
+        }
+    }
+}
--- a/buildSrc/src/main/groovy/io/airbyte/gradle/DockerGenerateConnectorBuildArgs.groovy
+++ b/buildSrc/src/main/groovy/io/airbyte/gradle/DockerGenerateConnectorBuildArgs.groovy
@@ -0,0 +1,51 @@
+package io.airbyte.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.*
+import org.yaml.snakeyaml.Yaml
+
+/**
+ * Reads `metadata.yaml` (if present) and emits
+ * `${buildDir}/docker/buildArgs.properties`
+ * so docker build tasks can consume per‑module arguments.
+ */
+class DockerGenerateConnectorBuildArgs extends DefaultTask {
+
+    @InputFile
+    final RegularFileProperty metadata = project.objects.fileProperty()
+
+    @OutputFile
+    final RegularFileProperty output = project.objects.fileProperty()
+
+    @Internal
+    final Property<Yaml> yaml = project.objects.property(Yaml).convention(new Yaml())
+
+    @TaskAction
+    void run() {
+        def metaFile = metadata.get().asFile
+        def outFile = output.get().asFile
+
+        if (!metaFile.exists()) {
+            outFile.text = ''          // keep Gradle’s cache happy
+            return
+        }
+
+        Map root = yaml.get().load(metaFile.text) ?: [:]
+        Map<String, ?> opts = ((root['data'] ?: [:])['connectorBuildOptions'] ?: [:]) as Map<String, ?>
+        outFile.withPrintWriter { pw ->
+            opts.each { k, v ->
+                if (v != null) {
+                    def key = k
+                            .replaceAll(/([a-z0-9])([A-Z])/, '$1_$2') // camelCase → snake_case
+                            .replace('-', '_')                        // dash → underscore
+                            .toUpperCase(Locale.ROOT)
+                    pw.println("${key}=$v")
+                }
+            }
+            /* Always add CONNECTOR_NAME=<module‑directory>. This is used to defined the image's name. */
+            pw.println("CONNECTOR_NAME=${project.projectDir.name}")
+        }
+    }
+}
--- a/docker-images/Dockerfile.java-connector-non-airbyte-ci
+++ b/docker-images/Dockerfile.java-connector-non-airbyte-ci
@@ -0,0 +1,30 @@
+# syntax=docker/dockerfile:1
+# check=skip=InvalidDefaultArgInFrom
+
+# Java connector Dockerfile for Airbyte
+
+# Build arguments
+ARG BASE_IMAGE
+
+# Base image - using Amazon Corretto (Amazon's distribution of OpenJDK)
+FROM ${BASE_IMAGE}
+ARG CONNECTOR_NAME
+
+# Set permissions for downloaded files
+RUN chmod +x /airbyte/base.sh /airbyte/javabase.sh && \
+    chown airbyte:airbyte /airbyte/base.sh /airbyte/javabase.sh /airbyte/dd-java-agent.jar
+
+ENV AIRBYTE_ENTRYPOINT="/airbyte/base.sh"
+ENV APPLICATION="${CONNECTOR_NAME}"
+
+# Add the connector TAR file and extract it
+COPY airbyte-app.tar /tmp/${CONNECTOR_NAME}.tar
+RUN tar xf /tmp/${CONNECTOR_NAME}.tar --strip-components=1 -C /airbyte && \
+    rm -rf /tmp/${CONNECTOR_NAME}.tar && \
+    chown -R airbyte:airbyte /airbyte
+
+# Set the non-root user
+USER airbyte
+
+# Set entrypoint
+ENTRYPOINT ["/airbyte/base.sh"]
--- a/settings.gradle
+++ b/settings.gradle
@@ -5,7 +5,9 @@ pluginManagement {
        // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.

        // ## Prefer repos controlled by Airbyte.
-        // TODO: add airbyte-controlled proxy repos here
+        maven {
+            url = 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
+        }

        // ## Look into other, public repos.
        // Gradle plugin portal.