chore: remove out-of-date/unused dagger JVM build code. (#64155)
Now that we have fully moved to gradle for JVM we can remove Python and airbyte-ci dependencies from the connectors build system to simplify Java connector builds and eliminate unnecessary Python toolchain requirements.
This commit is contained in:
Davin Chia
@@ -1,108 +1,5 @@
|
||||
plugins {
|
||||
id 'base'
|
||||
id 'ru.vyarus.use-python' version '2.3.0'
|
||||
id "io.airbyte.gradle.docker" version "0.63.0" apply false
|
||||
id 'com.github.eirnym.js2p' version "1.0" apply false
|
||||
}
|
||||
|
||||
// Pyenv support.
|
||||
try {
|
||||
def pyenvRoot = "pyenv root".execute()
|
||||
if (pyenvRoot.waitFor() == 0) {
|
||||
ext.pyenvRoot = pyenvRoot.text.trim()
|
||||
}
|
||||
} catch (IOException _) {
|
||||
// Swallow exception if pyenv is not installed.
|
||||
}
|
||||
|
||||
def pythonBin = layout.buildDirectory.file('.venv/bin/python').get().asFile.absolutePath
|
||||
|
||||
// python is required by the connectors project to run airbyte-ci from source to build connector images.
|
||||
python {
|
||||
envPath = layout.buildDirectory.file('.venv').get().asFile
|
||||
minPythonVersion = '3.11' // should be 3.11 for local development
|
||||
|
||||
// Pyenv support.
|
||||
try {
|
||||
def pyenvRoot = "pyenv root".execute()
|
||||
def pyenvLatest = "pyenv latest ${minPythonVersion}".execute()
|
||||
// Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases.
|
||||
pythonBinary "python"
|
||||
if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) {
|
||||
pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin"
|
||||
}
|
||||
} catch (IOException _) {
|
||||
// Swallow exception if pyenv is not installed.
|
||||
}
|
||||
|
||||
scope = 'VIRTUALENV'
|
||||
installVirtualenv = true
|
||||
// poetry is required for installing and running airbyte-ci
|
||||
pip 'poetry:1.5.1'
|
||||
}
|
||||
|
||||
def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) {
|
||||
workingDir rootProject.file('airbyte-ci/connectors/pipelines')
|
||||
commandLine pythonBin
|
||||
args "-m", "poetry", "install", "--no-cache"
|
||||
}
|
||||
poetryInstallAirbyteCI.configure {
|
||||
dependsOn tasks.named('pipInstall')
|
||||
}
|
||||
def poetryCleanVirtualenv = tasks.register('cleanPoetryVirtualenv', Exec) {
|
||||
workingDir rootProject.file('airbyte-ci/connectors/pipelines')
|
||||
commandLine pythonBin
|
||||
args "-m", "poetry", "env", "remove", "--all"
|
||||
onlyIf {
|
||||
layout.buildDirectory.file('venv/bin/python').get().asFile.exists()
|
||||
}
|
||||
}
|
||||
tasks.named('clean').configure {
|
||||
dependsOn poetryCleanVirtualenv
|
||||
}
|
||||
|
||||
allprojects {
|
||||
// Adds airbyte-ci task.
|
||||
def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs ->
|
||||
def task = tasks.register(taskName, Exec) {
|
||||
workingDir rootDir
|
||||
environment "CI", "1" // set to use more suitable logging format
|
||||
commandLine pythonBin
|
||||
args "-m", "poetry"
|
||||
args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}"
|
||||
args "run"
|
||||
args "airbyte-ci", "connectors", "--name=${project.name}"
|
||||
args connectorsArgs
|
||||
// Forbid these kinds of tasks from running concurrently.
|
||||
// We can induce serial execution by giving them all a common output directory.
|
||||
outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock")
|
||||
outputs.upToDateWhen { false }
|
||||
}
|
||||
task.configure { dependsOn poetryInstallAirbyteCI }
|
||||
return task
|
||||
}
|
||||
|
||||
// This allows us to slowly migrate connectors between dagger and the new plugins.
|
||||
// This can be deleted once we no longer use Dagger.
|
||||
def buildConnectorImage = airbyteCIConnectorsTask(
|
||||
'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar')
|
||||
buildConnectorImage.configure {
|
||||
// Images for java projects always rely on the distribution tarball.
|
||||
dependsOn tasks.matching { it.name == 'distTar' }
|
||||
// Ensure that all files exist beforehand.
|
||||
dependsOn tasks.matching { it.name == 'generate' }
|
||||
}
|
||||
|
||||
afterEvaluate { Project p ->
|
||||
if (!plugins.hasPlugin('airbyte-connector-docker-convention')) {
|
||||
tasks.named('assemble').configure {
|
||||
// We may revisit the dependency on assemble but the dependency should always be on a base task.
|
||||
dependsOn buildConnectorImage
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience tasks for local airbyte-ci execution.
|
||||
airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build')
|
||||
airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test')
|
||||
}
|
||||
|
||||
@@ -36,77 +36,6 @@ set -euo pipefail
|
||||
|
||||
CONNECTORS_DIR="airbyte-integrations/connectors"
|
||||
|
||||
# Function to check if a connector is in the whitelist
|
||||
is_in_whitelist() {
|
||||
local connector="$1"
|
||||
case "$connector" in
|
||||
destination-azure-blob-storage|\
|
||||
destination-bigquery|\
|
||||
destination-csv|\
|
||||
destination-clickhouse|\
|
||||
destination-clickhouse-strict-encrypt|\
|
||||
destination-databricks|\
|
||||
destination-dev-null|\
|
||||
destination-dynamodb|\
|
||||
destination-elasticsearch-strict-encrypt|\
|
||||
destination-elasticsearch|\
|
||||
destination-gcs|\
|
||||
destination-kafka|\
|
||||
destination-local-json|\
|
||||
destination-mongodb-strict-encrypt|\
|
||||
destination-mongodb|\
|
||||
destination-mssql|\
|
||||
destination-mysql-strict-encrypt|\
|
||||
destination-mysql|\
|
||||
destination-oracle-strict-encrypt|\
|
||||
destination-oracle|\
|
||||
destination-postgres-strict-encrypt|\
|
||||
destination-postgres|\
|
||||
destination-pubsub|\
|
||||
destination-redis|\
|
||||
destination-redshift|\
|
||||
destination-s3-data-lake|\
|
||||
destination-s3|\
|
||||
destination-singlestore|\
|
||||
destination-snowflake|\
|
||||
destination-starburst-galaxy|\
|
||||
destination-teradata|\
|
||||
destination-yellowbrick|\
|
||||
source-e2e-test|\
|
||||
source-postgres|\
|
||||
source-mysql|\
|
||||
source-mssql|\
|
||||
source-mongodb|\
|
||||
source-db2-enterprise|\
|
||||
source-sap-hana|\
|
||||
source-netsuite|\
|
||||
source-oracle-enterprise|\
|
||||
source-bigquery|\
|
||||
source-clickhouse|\
|
||||
source-clickhouse-strict-encrypt|\
|
||||
source-cockroachdb|\
|
||||
source-db2|\
|
||||
source-dynamodb|\
|
||||
source-e2e-test-cloud|\
|
||||
source-elasticsearch|\
|
||||
source-kafka|\
|
||||
source-oracle|\
|
||||
source-oracle-strict-encrypt|\
|
||||
source-scaffold-java-jdbc|\
|
||||
source-sftp|\
|
||||
source-snowflake|\
|
||||
source-teradata|\
|
||||
source-tidb|\
|
||||
source-redshift|\
|
||||
source-singlestore)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ------ Defaults & arg parsing -------
|
||||
publish_mode="pre-release"
|
||||
do_publish=false
|
||||
@@ -209,12 +138,6 @@ generate_dev_tag() {
|
||||
|
||||
# ---------- main loop ----------
|
||||
while read -r connector; do
|
||||
# only publish if connector is in whitelist
|
||||
if ! is_in_whitelist "$connector"; then
|
||||
echo "ℹ️ Skipping '$connector'; not in rollout whitelist"
|
||||
continue
|
||||
fi
|
||||
|
||||
meta="${CONNECTORS_DIR}/${connector}/metadata.yaml"
|
||||
if [[ ! -f "$meta" ]]; then
|
||||
echo "Error: metadata.yaml not found for ${connector}" >&2
|
||||
|
||||
Reference in New Issue
Block a user