diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 48e9b717968..2043c74f1f9 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -30,8 +30,8 @@ subprojects { subproject -> } repositories { maven { - name 'cloudrepo' - url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars' + name 'airbyte-public-jars' + url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' credentials { username System.getenv('CLOUDREPO_USER') password System.getenv('CLOUDREPO_PASSWORD') diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index a3539ab8616..9305d758091 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -40,14 +40,3 @@ dependencies { integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') } - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } -} diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index a839ed8e466..1c8d32b9d95 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -37,14 +37,3 @@ dependencies { testImplementation libs.testcontainers.elasticsearch integrationTestJavaImplementation libs.testcontainers.elasticsearch } - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } -} diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index f08e805a00d..b4efe934f0f 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -18,12 +18,6 @@ application { '-XX:GCLockerRetryAllocationCount=100',] } -repositories { - maven { - url "https://s3.amazonaws.com/redshift-maven-repository/release" - } -} - dependencies { implementation project(':airbyte-integrations:bases:base-typing-deduping') testImplementation project(':airbyte-integrations:bases:base-typing-deduping-test') diff --git a/airbyte-integrations/connectors/destination-rockset/build.gradle b/airbyte-integrations/connectors/destination-rockset/build.gradle index 4fc845d4dc4..1ecdf31062f 100644 --- a/airbyte-integrations/connectors/destination-rockset/build.gradle +++ b/airbyte-integrations/connectors/destination-rockset/build.gradle @@ -16,11 +16,6 @@ application { applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -repositories { - maven { url "https://mvnrepository.com/artifact/com.rockset/rockset-java" } - maven { url "https://mvnrepository.com/artifact/org.awaitility/awaitility" } -} - dependencies { implementation project(':airbyte-config-oss:config-models-oss') implementation libs.airbyte.protocol diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle index 0225d46c0e1..b808aec21fe 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/source-elasticsearch/build.gradle @@ -37,14 +37,3 @@ dependencies { testImplementation libs.testcontainers.elasticsearch integrationTestJavaImplementation libs.testcontainers.elasticsearch } - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } -} diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index 3c1fb80dc70..126636e8aec 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -16,12 +16,6 @@ application { applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -repositories { - maven { - url "https://packages.confluent.io/maven" - } -} - dependencies { implementation project(':airbyte-config-oss:config-models-oss') implementation libs.airbyte.protocol diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index bf9061ff400..6406eb19a09 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -16,10 +16,6 @@ application { applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -repositories { - maven { url "https://s3.amazonaws.com/redshift-maven-repository/release" } -} - dependencies { implementation libs.airbyte.protocol diff --git a/build.gradle b/build.gradle index 7075d6d32cd..54a66dd407f 100644 --- a/build.gradle +++ b/build.gradle @@ -7,24 +7,7 @@ import com.hierynomus.gradle.license.tasks.LicenseFormat // This is separate from application dependencies. // See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle. buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:8.0.0' - // 6.x version of OpenApi generator is only compatible with jackson-core 2.13.x onwards. - // This conflicts with the jackson depencneis the bmuschko plugin is pulling in. - // Since api generation is only used in the airbyte-api module and the base gradle files - // are loaded in first, Gradle is not able to intelligently resolve this before loading in - // the bmuschko plugin and thus placing an older jackson version on the class path. - // The alternative is to import the openapi plugin for all modules. - // This might need to be updated when we change openapi plugin versions. - classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0' - - classpath 'org.codehaus.groovy:groovy-yaml:3.0.3' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.20.0' classpath 'gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.16.1' } @@ -83,28 +66,6 @@ allprojects { project.base.archivesName = "${project.group}-${project.name}" version = rootProject.ext.version - - repositories { - // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. - - // Start with the local filesystem. - mavenLocal() - - // ## Look into repos controlled by Airbyte. - - // This repo hosts our public artifacts and can be referenced by anyone. - maven { url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' } - - // TODO: We might want to add a private proxy repo for maven central at some point. - - // ## Look into other, public repos. - - // Maven Central has most of everything. - mavenCentral() - - // Jitpack is used to pull dependencies whose package names start with 'com.github.' - maven { url 'https://jitpack.io' } - } } // License generation logic. @@ -149,9 +110,12 @@ node { download = true version = '18.18.0' npmVersion = '10.1.0' - // when setting both these directories, npm and node will be in separate directories + // When setting both these directories, npm and node will be in separate directories. workDir = file("${buildDir}/nodejs") npmWorkDir = file("${buildDir}/npm") + // Do not declare the repository. + distBaseUrl = null + } // python is required by the root project to apply python formatters like isort or black. diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 4fb7693fa0d..143a6597a40 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -3,6 +3,15 @@ plugins { } repositories { + // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. + + // ## Prefer repos controlled by Airbyte. + // TODO: add airbyte-controlled proxy repos here + + // ## Look into other, public repos. + // Gradle plugin portal. + gradlePluginPortal() + // Maven Central has most of everything. mavenCentral() } diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/settings.gradle b/settings.gradle index 72fcdddfdd7..903962ab182 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,22 +1,17 @@ import com.gradle.scan.plugin.PublishedBuildScan - pluginManagement { repositories { + // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. + + // ## Prefer repos controlled by Airbyte. + // TODO: add airbyte-controlled proxy repos here + + // ## Look into other, public repos. + // Gradle plugin portal. gradlePluginPortal() - maven { - url 'https://oss.sonatype.org/content/repositories/snapshots' - } - } - resolutionStrategy { - eachPlugin { - // We're using the 6.1.0-SNAPSHOT version of openapi-generator which contains a fix for generating nullable arrays (https://github.com/OpenAPITools/openapi-generator/issues/13025) - // The snapshot version isn't available in the main Gradle Plugin Portal, so we added the Sonatype snapshot repository above. - // The useModule command below allows us to map from the plugin id, `org.openapi.generator`, to the underlying module (https://oss.sonatype.org/content/repositories/snapshots/org/openapitools/openapi-generator-gradle-plugin/6.1.0-SNAPSHOT/_ - if (requested.id.id == 'org.openapi.generator') { - useModule "org.openapitools:openapi-generator-gradle-plugin:${requested.version}" - } - } + // Maven Central has most of everything. + mavenCentral() } } @@ -27,6 +22,109 @@ plugins { id 'com.github.burrunan.s3-build-cache' version "1.5" } +dependencyResolutionManagement { + // Set FAIL_ON_PROJECT_REPOS to ensure there are no more `repositories { ... }` blocks than necessary. + repositoriesMode = RepositoriesMode.FAIL_ON_PROJECT_REPOS + + repositories { + // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. + + // ## Start with the local filesystem. + mavenLocal() + + // ## Prefer repos controlled by Airbyte. + + maven { + // This repo hosts our public artifacts and can be referenced by anyone. + name 'airbyte-public-jars' + url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' + content { + // Whitelist artifacts served by this repo because it's slow. + includeGroup 'io.airbyte' + includeGroupByRegex 'io\\.airbyte\\..*' + includeGroup 'com.hadoop.gplcompression' + includeGroup 'com.therealvan' + } + } + // TODO: add airbyte-controlled proxy repos here + + // ## Look into other, public repos. + + // Maven Central has most of everything. + mavenCentral() + // Jitpack is used to pull dependencies directly from github. + maven { + name 'jitpack' + url 'https://jitpack.io' + content { + includeGroupByRegex 'com\\.github\\..*' + includeGroup 'net.jimblackler.jsonschemafriend' + } + } + // Elastic Search repo. + maven { + name 'elastic-search-snapshots' + url 'https://snapshots.elastic.co/maven/' + content { + includeGroup 'co.elastic.clients' + } + } + // Redshift repo. + maven { + name 'redshift' + url 'https://s3.amazonaws.com/redshift-maven-repository/release' + content { + includeGroup 'com.amazon.redshift' + } + } + // Rockset repo. + maven { + name 'rockset' + url 'https://mvnrepository.com/artifact/com.rockset/rockset-java' + content { + includeGroup 'com.rockset' + } + } + // Awaitility repo. + maven { + name 'awaitility' + url 'https://mvnrepository.com/artifact/org.awaitility/awaitility' + content { + includeGroup 'org.awaitility' + } + } + // Confluent repo. + maven { + name 'confluent' + url "https://packages.confluent.io/maven" + content { + includeGroup 'io.confluent' + includeGroup 'org.apache.kafka' + } + } + // Node.js download repository, required only by com.github.node-gradle.node plugin. + ivy { + name = "Node.js" + setUrl("https://nodejs.org/dist/") + patternLayout { + artifact("v[revision]/[artifact](-v[revision]-[classifier]).[ext]") + } + metadataSources { + artifact() + } + content { + includeModule("org.nodejs", "node") + } + } + } + + versionCatalogs { + libs { + from(files("deps.toml")) + } + } +} + gradleEnterprise { buildScan { termsOfServiceUrl = "https://gradle.com/terms-of-service" @@ -37,8 +135,6 @@ gradleEnterprise { } } - - ext.isCiServer = System.getenv().containsKey("CI") ext.isAirbyteCI = System.getenv().containsKey("RUN_IN_AIRBYTE_CI") @@ -63,33 +159,6 @@ if (isCiServer || isAirbyteCI) { rootProject.name = 'airbyte' -// definition for dependency resolution -dependencyResolutionManagement { - repositories { - // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. - - // Start with the local filesystem. - mavenLocal() - - // ## Look into repos controlled by Airbyte. - - // This repo hosts our public artifacts and can be referenced by anyone. - maven { url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' } - - // TODO: We might want to add a private proxy repo for maven central at some point. - - // ## Look into other, public repos. - - // Maven Central has most of everything. - mavenCentral() - } - versionCatalogs { - libs { - from(files("deps.toml")) - } - } -} - include ':airbyte-commons' include ':airbyte-api' include ':airbyte-commons-cli'