airbyte/settings.gradle

pluginManagement {
    repositories {
        // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.

        // ## Prefer repos controlled by Airbyte.
        maven {
            url = 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
        }

        // ## Look into other, public repos.
        // Gradle plugin portal.
        gradlePluginPortal()
        // Maven Central has most of everything.
        mavenCentral()
    }
}

// Configure the gradle enterprise plugin to enable build scans. Enabling the plugin at the top of the settings file allows the build scan to record
// as much information as possible.
plugins {
    id "com.gradle.develocity" version "3.19.2"
    id 'com.github.burrunan.s3-build-cache' version "1.8.1"
}

final boolean isCI = System.getenv().containsKey("CI") || System.getenv().containsKey("RUN_IN_AIRBYTE_CI")

dependencyResolutionManagement {
    // Set FAIL_ON_PROJECT_REPOS to ensure there are no more `repositories { ... }` blocks than necessary.
    repositoriesMode = RepositoriesMode.FAIL_ON_PROJECT_REPOS

    repositories {
        // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order.

        // ## Prefer repos controlled by Airbyte.

        maven {
            // This repo hosts our public artifacts and can be referenced by anyone.
            name 'airbyte-public-jars'
            url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
            content {
                // Whitelist artifacts served by this repo because it's slow.
                includeGroup 'io.airbyte'
                includeGroupByRegex 'io\\.airbyte\\..*'
                includeGroup 'com.hadoop.gplcompression'
                includeGroup 'com.therealvan'
            }
        }
        // TODO: add airbyte-controlled proxy repos here

        // ## Look into other, public repos.

        // Maven Central has most of everything.
        mavenCentral()
        // TODO remove this after jsonschemafriend is republished to jitpack https://github.com/jimblackler/jsonschemafriend/issues/124
        maven {
            name 'scijava'
            url 'https://maven.scijava.org/content/repositories/public'
            content {
                includeGroup 'net.jimblackler.jsonschemafriend'
            }
        }
        // Jitpack is used to pull dependencies directly from github.
        maven {
            name 'jitpack'
            url 'https://jitpack.io'
            content {
                includeGroupByRegex 'com\\.github\\..*'
                includeGroup 'net.jimblackler.jsonschemafriend'
            }
        }
        // Elastic Search repo.
        maven {
            name 'elastic-search-snapshots'
            url 'https://snapshots.elastic.co/maven/'
            content {
                includeGroup 'co.elastic.clients'
            }
        }
        // Redshift repo.
        maven {
            name 'redshift'
            url 'https://s3.amazonaws.com/redshift-maven-repository/release'
            content {
                includeGroup 'com.amazon.redshift'
            }
        }
        // Rockset repo.
        maven {
            name 'rockset'
            url 'https://mvnrepository.com/artifact/com.rockset/rockset-java'
            content {
                includeGroup 'com.rockset'
            }
        }
        // Awaitility repo.
        maven {
            name 'awaitility'
            url 'https://mvnrepository.com/artifact/org.awaitility/awaitility'
            content {
                includeGroup 'org.awaitility'
            }
        }
        // Confluent repo.
        maven {
            name 'confluent'
            url "https://packages.confluent.io/maven"
            content {
                includeGroup 'io.confluent'
                includeGroup 'org.apache.kafka'
            }
        }
    }
}

develocity {
    buildScan {
        termsOfUseUrl = "https://gradle.com/terms-of-service"
        termsOfUseAgree = "yes"
        uploadInBackground = !isCI // Disable in CI or scan URLs may not work.
        buildScanPublished { scan ->
            file("scan-journal.log") << "${new Date()} - ${scan.buildScanId} - ${scan.buildScanUri}\n"
        }
    }
}

if (isCI) {
    buildCache {
        local {
            // Local build cache is dangerous as it might produce inconsistent results
            // in case developer modifies files while the build is running
            enabled = true
            push = true
        }
        remote(com.github.burrunan.s3cache.AwsS3BuildCache) {
            enabled = !System.getenv().getOrDefault("S3_BUILD_CACHE_ACCESS_KEY_ID", "").isEmpty()
            push = true
            // The 'us-west-2' region was chosen because it was close to our v0.6 dagger runners
            // we used to maintain in AWS, prior to migrating to v0.9 and to github runners.
            // This may in fact no longer be the best choice, who knows? It's not terrible, at least.
            region = 'us-west-2'
            bucket = 'ab-ci-cache'
            prefix = "${System.getProperty('s3BuildCachePrefix', 'connectors')}-ci-cache/"
        }
    }
}

rootProject.name = 'airbyte'

boolean hasGradleBuildFile(java.nio.file.Path path) {
    def buildFiles = file(path).list { file, name -> name.matches("build\\.gradle(?:\\.kts)?") }
    return buildFiles.length == 1
}

void includeCdkProjects(String dir, String gradlePathPrefix, String projectNamePrefix) {
    def path = rootDir.toPath().resolve(dir)
    if (!path.toFile().exists()) {
        path = rootDir.toPath().resolve("airbyte-submodule").resolve(dir)
        if (!path.toFile().exists()) {
            return
        }
    }
    path.eachDir { subDir ->
        if (!hasGradleBuildFile(subDir)) {
            return
        }
        String gradlePath = gradlePathPrefix + subDir.getFileName()
        include gradlePath
        project(gradlePath).name = projectNamePrefix + subDir.getFileName()
    }
}

// Include convenience aggregator.
include ":airbyte-cdk"
// Include all java CDK modules.
includeCdkProjects("airbyte-cdk/java/airbyte-cdk", ":airbyte-cdk:java:airbyte-cdk:", "airbyte-cdk-")
includeCdkProjects("airbyte-cdk/bulk/core", ":airbyte-cdk:bulk:core:", "bulk-cdk-core-")
includeCdkProjects("airbyte-cdk/bulk/toolkits", ":airbyte-cdk:bulk:toolkits:", "bulk-cdk-toolkit-")

// Include all java connector projects.
def integrationsPath = rootDir.toPath().resolve('airbyte-integrations/connectors')
integrationsPath.eachDir { dir ->
    if (!hasGradleBuildFile(dir)) {
        // Ignore python and other non-gradle connectors.
        return
    }
    File metadataFile = dir.resolve("metadata.yaml").toFile()
    if (!metadataFile.exists()) {
        // Don't support connectors without metadata.
        return
    }
    String metadataYaml = metadataFile.getText("UTF-8")
    if (metadataYaml =~ /(?m)^\s+supportLevel:\s*["']?archived["']?\s*$/) {
        // Ignore archived connectors.
        return
    }
    include ":airbyte-integrations:connectors:${dir.getFileName()}"
}

// Include miscellaneous modules.
include ':airbyte-integrations:connectors-performance:source-harness'
include ':airbyte-integrations:connectors-performance:destination-harness'