chore: non-dagger builds. (#59716)

Introduce the airbyte gradle docker plugin to build JVM connectors.
Add logic to allow us to slowly migrate from one build set up to the other.

The majority of the work is taking in the metadata.yaml file, and converting that to buildArgs to be injected into the airbyte gradle docker plugin.

In particular, we extract the base image from the metadata file. We also add the connector name by referencing the module. This outputs a file buildArgs.properties into the build folder. This is done by the DockerGenerateConnectorBuildArgs task.

The AirbyteDockerConventionPlugin then takes in the generated buildArgs.properties file and feeds it into the Platform Docker plugin. It also feeds in the top-level Dockerfile, so the Docker plugin can copy it into the build folder.

I tried to migrate this to Kotlin, however the repo needs to be upgraded to the same Kotlin and KSP version as the gradle plugin repos for this to be worth while. Otherwise we end up writing kotlin plugins with reflection, which is worse than the current groovy plugins.

buildSrc/build.gradle

@@ -3,10 +3,15 @@ plugins {
 }
 
 repositories {
-    // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.
+    // Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.
     gradlePluginPortal()
 }
 
+dependencies {
+    // Note this dependency version needs to line up with the generateJsonSchema2Pojo plugin used in the project.
+    implementation 'org.yaml:snakeyaml:1.33'
+}
+
 tasks.withType(Jar).configureEach {
     duplicatesStrategy DuplicatesStrategy.INCLUDE
 }

buildSrc/src/main/groovy/airbyte-connector-docker-convention.gradle

@@ -0,0 +1,85 @@
+import io.airbyte.gradle.DockerGenerateConnectorBuildArgs
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.file.RegularFile
+import org.gradle.api.plugins.ExtensionAware
+import org.gradle.api.provider.Provider
+
+class AirbyteConnectorDockerConventionPlugin implements Plugin<Project> {
+    private static final String SHARED_DOCKERFILE = 'docker-images/Dockerfile.java-connector-non-airbyte-ci'
+
+    @Override
+    void apply(Project project) {
+        /*
+            We deliberately avoid importing classes from the published Airbyte Docker
+            plugin. That keeps buildSrc free of a compile‑time dependency on a specific
+            plugin version and lets this convention plugin compile before external
+            plugins are resolved.
+         */
+
+        project.plugins.withId('io.airbyte.gradle.docker') {
+            def dockerExt = ((project.extensions.findByName('airbyte') as ExtensionAware)
+                    ?.extensions?.findByName('docker'))
+
+            if (!dockerExt) {
+                project.logger.warn('airbyte.docker extension not found; skipping convention plugin.')
+                return
+            }
+
+            /* ------------------------------------------------------------
+             * 1. imageName default = project.name (if not already set)
+             * ------------------------------------------------------------ */
+            if (!dockerExt.imageName.present) {
+                dockerExt.imageName.convention(project.name)
+            }
+
+            /* ------------------------------------------------------------
+             * 2. Use shared Dockerfile if not overridden. This allows
+             *    users to specify the Dockerfiles per connector.
+             *    The core plugin’s dockerCopyDockerfile task will copy
+             *    it into build/airbyte/docker/Dockerfile automatically.
+             * ------------------------------------------------------------ */
+            if (!dockerExt.dockerfile.present) {
+                dockerExt.dockerfile.set(project.rootProject.file(SHARED_DOCKERFILE))
+            }
+
+            /* ------------------------------------------------------------
+             * 3. Task: generate buildArgs.properties from metadata.yaml
+             * ------------------------------------------------------------ */
+            def genProps = project.tasks.register(
+                    'generateConnectorDockerBuildArgs',
+                    DockerGenerateConnectorBuildArgs
+            ) {
+                metadata.set(project.layout.projectDirectory.file('metadata.yaml'))
+                output.set(project.layout.buildDirectory.file('docker/buildArgs.properties'))
+            }
+
+            /* ------------------------------------------------------------
+             * 4. Lazy provider: read and inject the file generated by the
+             *    DockerGenerateConnectorBuildArgs task as buildArgs
+             * ------------------------------------------------------------ */
+            def propsMapProvider = project.providers
+                    .fileContents(genProps.flatMap { it.output } as Provider<RegularFile>)
+                    .asText
+                    .map { txt ->
+                        txt.readLines()
+                                .findAll { it.contains('=') }
+                                .collectEntries { ln ->
+                                    def (k, v) = ln.split('=', 2)
+                                    [(k.trim()): v.trim()]
+                                }
+                    }
+            dockerExt.buildArgs.putAll(propsMapProvider)
+
+            /* ------------------------------------------------------------
+             * 5. Make dockerBuildx depend on the generator, and register
+             *    the properties file as an input (incremental build‑cache)
+             * ------------------------------------------------------------ */
+            project.tasks.named('dockerBuildx').configure { Task t ->
+                t.dependsOn(genProps)
+                t.inputs.file(genProps.flatMap { it.output })
+            }
+        }
+    }
+}

buildSrc/src/main/groovy/io/airbyte/gradle/DockerGenerateConnectorBuildArgs.groovy

@@ -0,0 +1,51 @@
+package io.airbyte.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.*
+import org.yaml.snakeyaml.Yaml
+
+/**
+ * Reads `metadata.yaml` (if present) and emits
+ * `${buildDir}/docker/buildArgs.properties`
+ * so docker build tasks can consume per‑module arguments.
+ */
+class DockerGenerateConnectorBuildArgs extends DefaultTask {
+
+    @InputFile
+    final RegularFileProperty metadata = project.objects.fileProperty()
+
+    @OutputFile
+    final RegularFileProperty output = project.objects.fileProperty()
+
+    @Internal
+    final Property<Yaml> yaml = project.objects.property(Yaml).convention(new Yaml())
+
+    @TaskAction
+    void run() {
+        def metaFile = metadata.get().asFile
+        def outFile = output.get().asFile
+
+        if (!metaFile.exists()) {
+            outFile.text = ''          // keep Gradle’s cache happy
+            return
+        }
+
+        Map root = yaml.get().load(metaFile.text) ?: [:]
+        Map<String, ?> opts = ((root['data'] ?: [:])['connectorBuildOptions'] ?: [:]) as Map<String, ?>
+        outFile.withPrintWriter { pw ->
+            opts.each { k, v ->
+                if (v != null) {
+                    def key = k
+                            .replaceAll(/([a-z0-9])([A-Z])/, '$1_$2') // camelCase → snake_case
+                            .replace('-', '_')                        // dash → underscore
+                            .toUpperCase(Locale.ROOT)
+                    pw.println("${key}=$v")
+                }
+            }
+            /* Always add CONNECTOR_NAME=<module‑directory>. This is used to defined the image's name. */
+            pw.println("CONNECTOR_NAME=${project.projectDir.name}")
+        }
+    }
+}