73034c64da
* sweep all scheduler application code and new-scheduler conditional logic * remove airbyte-scheduler from deployments and docs * format * remove 'v2' from github actions * add back scheduler in delete deployment command * remove scheduler parameters from helm chart values * add back job cleaner + test and add comment * remove now-unused env vars from code and docs * format * remove feature flags from web backend connection handler as it is no longer needed * remove feature flags from config api as it is now longer needed * remove feature flags input from config api test * format + shorter url * remove scheduler parameters from helm chart readme
502 lines
16 KiB
Groovy
502 lines
16 KiB
Groovy
import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
|
|
import com.github.spotbugs.snom.SpotBugsTask
|
|
|
|
buildscript {
|
|
repositories {
|
|
maven {
|
|
url "https://plugins.gradle.org/m2/"
|
|
}
|
|
}
|
|
dependencies {
|
|
classpath 'com.bmuschko:gradle-docker-plugin:7.2.0'
|
|
}
|
|
}
|
|
|
|
plugins {
|
|
id 'base'
|
|
id 'pmd'
|
|
id 'com.diffplug.spotless' version '6.0.0'
|
|
id 'com.github.hierynomus.license' version '0.16.1'
|
|
id 'com.github.spotbugs' version '5.0.6'
|
|
id 'version-catalog'
|
|
id 'maven-publish'
|
|
}
|
|
|
|
apply from: "$rootDir/publish-repositories.gradle"
|
|
|
|
repositories {
|
|
mavenCentral()
|
|
}
|
|
|
|
Properties env = new Properties()
|
|
rootProject.file('.env').withInputStream { env.load(it) }
|
|
|
|
if (!env.containsKey('VERSION')) {
|
|
throw new Exception('Version not specified in .env file...')
|
|
}
|
|
|
|
// `version` is used as the application build version for artifacts like jars
|
|
// `image_tag` is used as the docker tag applied to built images.
|
|
// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
|
|
// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
|
|
ext {
|
|
version = System.getenv("VERSION") ?: env.VERSION
|
|
image_tag = System.getenv("VERSION") ?: 'dev'
|
|
}
|
|
|
|
def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
|
|
/*
|
|
In java, we don't have a second linter/styling tool other than spotless so it doesn't really
|
|
matter if we write a newline or not for startComment/endComment.
|
|
|
|
However, in python, we are using black that double-checks and reformats the code.
|
|
Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a
|
|
big deal and would be reformatted (removed) because of black's specs.
|
|
*/
|
|
def tmp = File.createTempFile('tmp', '.tmp')
|
|
tmp.withWriter {
|
|
def w = it
|
|
if (startComment.length() > 0 || !isPython) {
|
|
w.writeLine(startComment)
|
|
}
|
|
license.eachLine {
|
|
w << lineComment
|
|
w.writeLine(it)
|
|
}
|
|
if (endComment.length() > 0 || !isPython) {
|
|
w.writeLine(endComment)
|
|
}
|
|
w.writeLine("")
|
|
if (isPython) {
|
|
w.writeLine("")
|
|
}
|
|
}
|
|
return tmp
|
|
}
|
|
|
|
def createPythonLicenseWith = { license ->
|
|
return createLicenseWith(license, '', '', '', true)
|
|
}
|
|
|
|
def createJavaLicenseWith = { license ->
|
|
return createLicenseWith(license, '/*', ' */', ' * ', false)
|
|
}
|
|
|
|
// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a
|
|
// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable.
|
|
def createSpotlessTarget = { pattern ->
|
|
def excludes = [
|
|
'.gradle',
|
|
'node_modules',
|
|
'.eggs',
|
|
'.mypy_cache',
|
|
'.venv',
|
|
'*.egg-info',
|
|
'build',
|
|
'dbt-project-template',
|
|
'dbt-project-template-mssql',
|
|
'dbt-project-template-mysql',
|
|
'dbt-project-template-oracle',
|
|
'dbt-project-template-clickhouse',
|
|
'dbt-project-template-snowflake',
|
|
'dbt_test_config',
|
|
'normalization_test_output',
|
|
'tools',
|
|
'secrets',
|
|
'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately.
|
|
'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file.
|
|
'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad
|
|
]
|
|
|
|
if (System.getenv().containsKey("SUB_BUILD")) {
|
|
excludes.add("airbyte-integrations/connectors")
|
|
}
|
|
|
|
return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" })
|
|
}
|
|
|
|
spotless {
|
|
java {
|
|
target createSpotlessTarget('**/*.java')
|
|
|
|
importOrder()
|
|
|
|
eclipse('4.21.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))
|
|
|
|
licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT'))
|
|
removeUnusedImports()
|
|
trimTrailingWhitespace()
|
|
}
|
|
groovyGradle {
|
|
target createSpotlessTarget('**/*.gradle')
|
|
}
|
|
sql {
|
|
target createSpotlessTarget('**/*.sql')
|
|
|
|
dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
|
|
}
|
|
format 'styling', {
|
|
target createSpotlessTarget(['**/*.yaml', '**/*.json'])
|
|
|
|
prettier()
|
|
}
|
|
}
|
|
check.dependsOn 'spotlessApply'
|
|
|
|
@SuppressWarnings('GroovyAssignabilityCheck')
|
|
def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
|
|
return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
|
|
def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
|
|
|
|
def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase()
|
|
def isArm64 = arch == "aarch64" || arch == "arm64"
|
|
|
|
def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64'
|
|
def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14'
|
|
def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:1.19-alpine' : 'amd64/nginx:1.19-alpine'
|
|
def openjdkImage = System.getenv('JDK_IMAGE') ?: isArm64 ? "arm64v8/openjdk:${jdkVersion}-slim" : "amd64/openjdk:${jdkVersion}-slim"
|
|
def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64'
|
|
|
|
inputDir = file("$projectDir/build/docker")
|
|
platform = buildPlatform
|
|
images.add("airbyte/$artifactName:$buildTag")
|
|
buildArgs.put('JDK_VERSION', jdkVersion)
|
|
buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
|
|
buildArgs.put('ALPINE_IMAGE', alpineImage)
|
|
buildArgs.put('NGINX_IMAGE', nginxImage)
|
|
buildArgs.put('JDK_IMAGE', openjdkImage)
|
|
buildArgs.put('VERSION', buildVersion)
|
|
})
|
|
}
|
|
|
|
@SuppressWarnings('GroovyAssignabilityCheck')
|
|
def Task getPublishArtifactsTask(String buildVersion, Project subproject) {
|
|
// generate a unique task name based on the directory name.
|
|
return task ("publishArtifact-$subproject.name" {
|
|
apply plugin: 'maven-publish'
|
|
publishing {
|
|
repositories {
|
|
publications {
|
|
"$subproject.name"(MavenPublication) {
|
|
from subproject.components.java
|
|
|
|
// use the subproject group and name with the assumption there are no identical subproject
|
|
// names, group names or subproject group/name combination.
|
|
groupId = "$subproject.group"
|
|
artifactId = "$subproject.name"
|
|
version = "$buildVersion"
|
|
repositories.add(rootProject.repositories.getByName('cloudrepo'))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
allprojects {
|
|
apply plugin: 'com.bmuschko.docker-remote-api'
|
|
|
|
task copyDocker(type: Sync) {
|
|
from "${project.projectDir}/Dockerfile"
|
|
into "build/docker/"
|
|
}
|
|
}
|
|
|
|
allprojects {
|
|
apply plugin: 'base'
|
|
|
|
// by default gradle uses directory as the project name. That works very well in a single project environment but
|
|
// projects clobber each other in an environments with subprojects when projects are in directories named identically.
|
|
def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.')
|
|
group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
|
|
project.archivesBaseName = "${project.group}-${project.name}"
|
|
|
|
version = rootProject.ext.version
|
|
}
|
|
|
|
// Java projects common configurations
|
|
subprojects {
|
|
|
|
if (project.name == 'airbyte-webapp' || project.name == 'airbyte-webapp-e2e-tests') {
|
|
return
|
|
}
|
|
|
|
apply plugin: 'java'
|
|
apply plugin: 'jacoco'
|
|
apply plugin: 'com.github.spotbugs'
|
|
apply plugin: 'pmd'
|
|
|
|
sourceCompatibility = JavaVersion.VERSION_17
|
|
targetCompatibility = JavaVersion.VERSION_17
|
|
|
|
repositories {
|
|
mavenCentral()
|
|
maven {
|
|
// TODO(Issue-4915): Remove this when upstream is merged in.
|
|
url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/'
|
|
}
|
|
maven {
|
|
url 'https://jitpack.io'
|
|
}
|
|
}
|
|
|
|
pmd {
|
|
consoleOutput = true
|
|
ignoreFailures = false
|
|
rulesMinimumPriority = 5
|
|
ruleSets = []
|
|
ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml'))
|
|
toolVersion = '6.43.0'
|
|
}
|
|
|
|
jacoco {
|
|
toolVersion = "0.8.7"
|
|
}
|
|
|
|
jacocoTestReport {
|
|
dependsOn test
|
|
reports {
|
|
html.required = true
|
|
xml.required = true
|
|
csv.required = false
|
|
}
|
|
}
|
|
|
|
jacocoTestCoverageVerification {
|
|
violationRules {
|
|
failOnViolation = false
|
|
rule {
|
|
element = 'CLASS'
|
|
excludes = ['**/*Test*', '**/generated*']
|
|
limit {
|
|
counter = 'BRANCH'
|
|
minimum = 0.8
|
|
}
|
|
limit {
|
|
counter = 'INSTRUCTION'
|
|
minimum = 0.8
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def integrationTagName = 'platform-integration'
|
|
def slowIntegrationTagName = 'platform-slow-integration'
|
|
// make tag accessible in submodules.
|
|
ext {
|
|
cloudStorageTestTagName = 'cloud-storage'
|
|
}
|
|
|
|
spotbugs {
|
|
ignoreFailures = false
|
|
effort = 'max'
|
|
excludeFilter = rootProject.file('spotbugs-exclude-filter-file.xml')
|
|
reportLevel = 'high'
|
|
showProgress = false
|
|
toolVersion = '4.6.0'
|
|
}
|
|
|
|
test {
|
|
jacoco {
|
|
enabled = true
|
|
excludes = ['**/*Test*', '**/generated*']
|
|
}
|
|
useJUnitPlatform {
|
|
excludeTags(integrationTagName, slowIntegrationTagName, cloudStorageTestTagName)
|
|
}
|
|
testLogging() {
|
|
events 'failed'
|
|
exceptionFormat 'full'
|
|
// showStandardStreams = true
|
|
}
|
|
finalizedBy jacocoTestReport
|
|
}
|
|
|
|
task newIntegrationTests(type: Test) {
|
|
useJUnitPlatform {
|
|
includeTags integrationTagName
|
|
}
|
|
testLogging() {
|
|
events 'failed'
|
|
exceptionFormat 'full'
|
|
}
|
|
finalizedBy jacocoTestReport
|
|
}
|
|
|
|
task slowIntegrationTests(type: Test) {
|
|
useJUnitPlatform {
|
|
includeTags slowIntegrationTagName
|
|
}
|
|
testLogging() {
|
|
events 'failed'
|
|
exceptionFormat 'full'
|
|
}
|
|
finalizedBy jacocoTestReport
|
|
}
|
|
|
|
task allTests(type: Test) {
|
|
useJUnitPlatform()
|
|
testLogging() {
|
|
events 'failed'
|
|
exceptionFormat 'full'
|
|
}
|
|
finalizedBy jacocoTestReport
|
|
}
|
|
|
|
dependencies {
|
|
if (project.name != 'airbyte-commons') {
|
|
implementation project(':airbyte-commons')
|
|
}
|
|
|
|
implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0"))
|
|
implementation(platform("org.glassfish.jersey:jersey-bom:2.31"))
|
|
|
|
|
|
// version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here.
|
|
implementation libs.bundles.jackson
|
|
|
|
implementation libs.guava
|
|
|
|
implementation libs.commons.io
|
|
|
|
implementation libs.bundles.apache
|
|
|
|
implementation libs.slf4j.api
|
|
|
|
|
|
// SLF4J as a facade over Log4j2 required dependencies
|
|
implementation libs.bundles.log4j
|
|
|
|
// Bridges from other logging implementations to SLF4J
|
|
implementation libs.bundles.slf4j
|
|
|
|
// Dependencies for logging to cloud storage, as well as the various clients used to do so.
|
|
implementation libs.appender.log4j2
|
|
implementation libs.aws.java.sdk.s3
|
|
implementation libs.google.cloud.storage
|
|
|
|
implementation libs.s3
|
|
|
|
// Lombok dependencies
|
|
compileOnly libs.lombok
|
|
annotationProcessor libs.lombok
|
|
|
|
testCompileOnly libs.lombok
|
|
testAnnotationProcessor libs.lombok
|
|
|
|
testRuntimeOnly libs.junit.jupiter.engine
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.assertj.core
|
|
|
|
testImplementation libs.junit.pioneer
|
|
|
|
// adds owasp plugin
|
|
spotbugsPlugins libs.findsecbugs.plugin
|
|
implementation libs.spotbugs.annotations
|
|
}
|
|
|
|
tasks.withType(Tar) {
|
|
duplicatesStrategy DuplicatesStrategy.INCLUDE
|
|
}
|
|
|
|
tasks.withType(Zip) {
|
|
duplicatesStrategy DuplicatesStrategy.INCLUDE
|
|
}
|
|
|
|
tasks.withType(SpotBugsTask) {
|
|
// Reports can be found under each subproject in build/spotbugs/
|
|
reports {
|
|
xml.required = false
|
|
html.required = true
|
|
}
|
|
}
|
|
|
|
tasks.withType(Pmd) {
|
|
exclude '**/generated/**'
|
|
exclude '**/jooq/**'
|
|
}
|
|
|
|
javadoc.options.addStringOption('Xdoclint:none', '-quiet')
|
|
check.dependsOn 'jacocoTestCoverageVerification'
|
|
}
|
|
|
|
task('generate') {
|
|
dependsOn subprojects.collect { it.getTasksByName('generateProtocolClassFiles', true) }
|
|
dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) }
|
|
}
|
|
|
|
task('format') {
|
|
dependsOn generate
|
|
dependsOn spotlessApply
|
|
dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) }
|
|
}
|
|
|
|
// add licenses for python projects.
|
|
subprojects {
|
|
def pythonFormatTask = project.tasks.findByName('blackFormat')
|
|
|
|
if (pythonFormatTask != null) {
|
|
apply plugin: "com.github.hierynomus.license"
|
|
license {
|
|
header rootProject.file("LICENSE_SHORT")
|
|
}
|
|
task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) {
|
|
header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT'))
|
|
source = fileTree(dir: projectDir)
|
|
.include("**/*.py")
|
|
.exclude(".venv/**/*.py")
|
|
.exclude("**/airbyte_api_client/**/*.py")
|
|
.exclude("**/__init__.py")
|
|
strictCheck = true
|
|
}
|
|
def licenseTask = project.tasks.findByName('licenseFormatPython')
|
|
blackFormat.dependsOn licenseTask
|
|
isortFormat.dependsOn licenseTask
|
|
flakeCheck.dependsOn licenseTask
|
|
|
|
def generateFilesTask = project.tasks.findByName('generateProtocolClassFiles')
|
|
if (generateFilesTask != null) {
|
|
licenseTask.dependsOn generateFilesTask
|
|
}
|
|
}
|
|
}
|
|
|
|
task('generate-docker') {
|
|
dependsOn(':airbyte-bootloader:assemble')
|
|
dependsOn(':airbyte-workers:assemble')
|
|
dependsOn(':airbyte-webapp:assemble')
|
|
dependsOn(':airbyte-server:assemble')
|
|
dependsOn(':airbyte-db:db-lib:assemble')
|
|
dependsOn(':airbyte-config:init:assemble')
|
|
dependsOn(':airbyte-temporal:assemble')
|
|
}
|
|
|
|
// produce reproducible archives
|
|
// (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
|
|
tasks.withType(AbstractArchiveTask) {
|
|
preserveFileTimestamps = false
|
|
reproducibleFileOrder = true
|
|
}
|
|
|
|
// definition for publishing
|
|
catalog {
|
|
versionCatalog {
|
|
from(files("deps.toml"))
|
|
}
|
|
}
|
|
|
|
publishing {
|
|
publications {
|
|
// This block is present for dependency catalog publishing.
|
|
maven(MavenPublication) {
|
|
groupId = 'io.airbyte'
|
|
artifactId = 'oss-catalog'
|
|
|
|
from components.versionCatalog
|
|
// Gradle will by default use the subproject path as the group id and the subproject name as the artifact id.
|
|
// e.g. the subproject :airbyte-scheduler:scheduler-models is imported at io.airbyte.airbyte-config:config-persistence:<version-number>.
|
|
}
|
|
}
|
|
repositories.add(rootProject.repositories.getByName('cloudrepo'))
|
|
}
|