From 578cbb8a7f7b52cd25d951271e9d5ab4d3021e5b Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 1 Aug 2025 09:13:45 -0700 Subject: [PATCH] chore: remove out-of-date/unused dagger JVM build code. (#64155) Now that we have fully moved to gradle for JVM we can remove Python and airbyte-ci dependencies from the connectors build system to simplify Java connector builds and eliminate unnecessary Python toolchain requirements. --- airbyte-integrations/connectors/build.gradle | 103 ------------------ ...ld-and-publish-java-connectors-with-tag.sh | 77 ------------- 2 files changed, 180 deletions(-) diff --git a/airbyte-integrations/connectors/build.gradle b/airbyte-integrations/connectors/build.gradle index ab16b7e21ca..44c85d39a61 100644 --- a/airbyte-integrations/connectors/build.gradle +++ b/airbyte-integrations/connectors/build.gradle @@ -1,108 +1,5 @@ plugins { id 'base' - id 'ru.vyarus.use-python' version '2.3.0' id "io.airbyte.gradle.docker" version "0.63.0" apply false id 'com.github.eirnym.js2p' version "1.0" apply false } - -// Pyenv support. -try { - def pyenvRoot = "pyenv root".execute() - if (pyenvRoot.waitFor() == 0) { - ext.pyenvRoot = pyenvRoot.text.trim() - } -} catch (IOException _) { - // Swallow exception if pyenv is not installed. -} - -def pythonBin = layout.buildDirectory.file('.venv/bin/python').get().asFile.absolutePath - -// python is required by the connectors project to run airbyte-ci from source to build connector images. -python { - envPath = layout.buildDirectory.file('.venv').get().asFile - minPythonVersion = '3.11' // should be 3.11 for local development - - // Pyenv support. - try { - def pyenvRoot = "pyenv root".execute() - def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() - // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. - pythonBinary "python" - if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { - pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" - } - } catch (IOException _) { - // Swallow exception if pyenv is not installed. - } - - scope = 'VIRTUALENV' - installVirtualenv = true - // poetry is required for installing and running airbyte-ci - pip 'poetry:1.5.1' -} - -def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) { - workingDir rootProject.file('airbyte-ci/connectors/pipelines') - commandLine pythonBin - args "-m", "poetry", "install", "--no-cache" -} -poetryInstallAirbyteCI.configure { - dependsOn tasks.named('pipInstall') -} -def poetryCleanVirtualenv = tasks.register('cleanPoetryVirtualenv', Exec) { - workingDir rootProject.file('airbyte-ci/connectors/pipelines') - commandLine pythonBin - args "-m", "poetry", "env", "remove", "--all" - onlyIf { - layout.buildDirectory.file('venv/bin/python').get().asFile.exists() - } -} -tasks.named('clean').configure { - dependsOn poetryCleanVirtualenv -} - -allprojects { - // Adds airbyte-ci task. - def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs -> - def task = tasks.register(taskName, Exec) { - workingDir rootDir - environment "CI", "1" // set to use more suitable logging format - commandLine pythonBin - args "-m", "poetry" - args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}" - args "run" - args "airbyte-ci", "connectors", "--name=${project.name}" - args connectorsArgs - // Forbid these kinds of tasks from running concurrently. - // We can induce serial execution by giving them all a common output directory. - outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock") - outputs.upToDateWhen { false } - } - task.configure { dependsOn poetryInstallAirbyteCI } - return task - } - - // This allows us to slowly migrate connectors between dagger and the new plugins. - // This can be deleted once we no longer use Dagger. - def buildConnectorImage = airbyteCIConnectorsTask( - 'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar') - buildConnectorImage.configure { - // Images for java projects always rely on the distribution tarball. - dependsOn tasks.matching { it.name == 'distTar' } - // Ensure that all files exist beforehand. - dependsOn tasks.matching { it.name == 'generate' } - } - - afterEvaluate { Project p -> - if (!plugins.hasPlugin('airbyte-connector-docker-convention')) { - tasks.named('assemble').configure { - // We may revisit the dependency on assemble but the dependency should always be on a base task. - dependsOn buildConnectorImage - } - } - } - - // Convenience tasks for local airbyte-ci execution. - airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build') - airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test') -} diff --git a/poe-tasks/build-and-publish-java-connectors-with-tag.sh b/poe-tasks/build-and-publish-java-connectors-with-tag.sh index 8f265129015..a1e60fc3b4f 100755 --- a/poe-tasks/build-and-publish-java-connectors-with-tag.sh +++ b/poe-tasks/build-and-publish-java-connectors-with-tag.sh @@ -36,77 +36,6 @@ set -euo pipefail CONNECTORS_DIR="airbyte-integrations/connectors" -# Function to check if a connector is in the whitelist -is_in_whitelist() { - local connector="$1" - case "$connector" in - destination-azure-blob-storage|\ - destination-bigquery|\ - destination-csv|\ - destination-clickhouse|\ - destination-clickhouse-strict-encrypt|\ - destination-databricks|\ - destination-dev-null|\ - destination-dynamodb|\ - destination-elasticsearch-strict-encrypt|\ - destination-elasticsearch|\ - destination-gcs|\ - destination-kafka|\ - destination-local-json|\ - destination-mongodb-strict-encrypt|\ - destination-mongodb|\ - destination-mssql|\ - destination-mysql-strict-encrypt|\ - destination-mysql|\ - destination-oracle-strict-encrypt|\ - destination-oracle|\ - destination-postgres-strict-encrypt|\ - destination-postgres|\ - destination-pubsub|\ - destination-redis|\ - destination-redshift|\ - destination-s3-data-lake|\ - destination-s3|\ - destination-singlestore|\ - destination-snowflake|\ - destination-starburst-galaxy|\ - destination-teradata|\ - destination-yellowbrick|\ - source-e2e-test|\ - source-postgres|\ - source-mysql|\ - source-mssql|\ - source-mongodb|\ - source-db2-enterprise|\ - source-sap-hana|\ - source-netsuite|\ - source-oracle-enterprise|\ - source-bigquery|\ - source-clickhouse|\ - source-clickhouse-strict-encrypt|\ - source-cockroachdb|\ - source-db2|\ - source-dynamodb|\ - source-e2e-test-cloud|\ - source-elasticsearch|\ - source-kafka|\ - source-oracle|\ - source-oracle-strict-encrypt|\ - source-scaffold-java-jdbc|\ - source-sftp|\ - source-snowflake|\ - source-teradata|\ - source-tidb|\ - source-redshift|\ - source-singlestore) - return 0 - ;; - *) - return 1 - ;; - esac -} - # ------ Defaults & arg parsing ------- publish_mode="pre-release" do_publish=false @@ -209,12 +138,6 @@ generate_dev_tag() { # ---------- main loop ---------- while read -r connector; do - # only publish if connector is in whitelist - if ! is_in_whitelist "$connector"; then - echo "ℹ️ Skipping '$connector'; not in rollout whitelist" - continue - fi - meta="${CONNECTORS_DIR}/${connector}/metadata.yaml" if [[ ! -f "$meta" ]]; then echo "Error: metadata.yaml not found for ${connector}" >&2