jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

migrate everything from java 17 to java 21 (#35103)

Browse Source
This commit is contained in:
Marius Posta
2024-02-09 17:26:19 -08:00
committed by GitHub
parent fdb3030ca0
commit b9ab3d1a90
31 changed files with 69 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/runner-prepare-for-build/action.yml vendored
View File

@@ -20,7 +20,7 @@ runs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- if: inputs.install_node == 'true'
uses: actions/setup-node@v3

2
.github/workflows/connector-performance-command.yml vendored
View File

@@ -159,7 +159,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Python
uses: actions/setup-python@v4
with:

2
.github/workflows/gradle.yml vendored
View File

@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Pip
if: steps.changes.outputs.gradlecheck_any_changed == 'true'
run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3

2
.github/workflows/jacoco_report.yml vendored
View File

@@ -16,7 +16,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Publish CDK to MavenLocal
run: |

2
.github/workflows/legacy-publish-command.yml vendored
View File

@@ -244,7 +244,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Python
uses: actions/setup-python@v4
with:

2
.github/workflows/legacy-test-command.yml vendored
View File

@@ -92,7 +92,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Python
uses: actions/setup-python@v4
with:

4
.github/workflows/publish-cdk-command-manually.yml vendored
View File

@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
@@ -228,7 +228,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Python
uses: actions/setup-python@v4
with:

2
.github/workflows/publish-java-cdk-command.yml vendored
View File

@@ -116,7 +116,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }})
if: ${{ !(env.FORCE == 'true') }}
run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished

4
.github/workflows/release-airbyte-os.yml vendored
View File

@@ -49,7 +49,7 @@ jobs:
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- uses: actions/setup-node@v3
with:
@@ -184,7 +184,7 @@ jobs:
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- uses: actions/setup-python@v4
with:

2
.github/workflows/test-performance-command.yml vendored
View File

@@ -84,7 +84,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "17"
java-version: "21"
- name: Install Python
uses: actions/setup-python@v4
with:

2
airbyte-cdk/java/airbyte-cdk/core/build.gradle
View File

@@ -1,7 +1,7 @@
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads"
options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-this-escape"
}
compileTestJava {
options.compilerArgs += "-Xlint:-try,-divzero,-cast"

7
airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle
View File

@@ -1,3 +1,10 @@
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-deprecation,-this-escape"
}
}
dependencies {
implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies')
implementation project(':airbyte-cdk:java:airbyte-cdk:core')

4
airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle
View File

@@ -1,10 +1,10 @@
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-deprecation,-removal"
options.compilerArgs += "-Xlint:-deprecation,-removal,-this-escape"
}
compileTestFixturesJava {
options.compilerArgs += "-Xlint:-try"
options.compilerArgs += "-Xlint:-try,-this-escape"
}
}

2
airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle
View File

@@ -7,7 +7,7 @@ plugins {
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal"
options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal,-this-escape"
}
}

2
airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle
View File

@@ -10,7 +10,7 @@ plugins {
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-varargs,-try,-deprecation,-unchecked"
options.compilerArgs += "-Xlint:-varargs,-try,-deprecation,-unchecked,-this-escape"
}
compileTestJava {
options.compilerArgs += "-Xlint:-try"

2
airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml
View File

@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO" packages="io.airbyte.commons.logging">
<Configuration status="INFO">
<Properties>
<!-- Set the LOG_SCRUB_PATTERN env var to a regex pattern to scrub the log messages of secrets or any other unwanted content. -->
<Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>

2
airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java
View File

@@ -83,8 +83,6 @@ class TestHarnessUtilsTest {
while (recordedBeats.get() < 3) {
Thread.sleep(10);
}
thread.stop();
}
@Test

2
airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle
View File

@@ -1,7 +1,7 @@
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-try,-deprecation"
options.compilerArgs += "-Xlint:-try,-deprecation,-this-escape"
}
compileTestJava {
options.compilerArgs += "-Xlint:-try"

1
airbyte-ci/connectors/pipelines/README.md
View File

@@ -640,6 +640,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
| Version | PR | Description |
| ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
| 4.2.0 | [#35103](https://github.com/airbytehq/airbyte/pull/35103) | Java 21 support. |
| 4.1.4 | [#35039](https://github.com/airbytehq/airbyte/pull/35039) | Fix bug which prevented gradle test reports from being added. |
| 4.1.3 | [#35010](https://github.com/airbytehq/airbyte/pull/35010) | Use `poetry install --no-root` in the builder container. |
| 4.1.2 | [#34945](https://github.com/airbytehq/airbyte/pull/34945) | Only install main dependencies when running poetry install. |

6
airbyte-ci/connectors/pipelines/pipelines/consts.py
View File

@@ -30,12 +30,12 @@ PLATFORM_MACHINE_TO_DAGGER_PLATFORM = {
}
LOCAL_MACHINE_TYPE = platform.machine()
LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[LOCAL_MACHINE_TYPE]
AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023"
AMAZONCORRETTO_IMAGE = "amazoncorretto:21-al2023"
NODE_IMAGE = "node:18.18.0-slim"
GO_IMAGE = "golang:1.17"
PYTHON_3_10_IMAGE = "python:3.10.13-slim"
MAVEN_IMAGE = "maven:3.9.5-amazoncorretto-17-al2023"
DOCKER_VERSION = "24.0.2"
MAVEN_IMAGE = "maven:3.9.6-amazoncorretto-21-al2023"
DOCKER_VERSION = "24"
DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind"
DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli"
DOCKER_REGISTRY_MIRROR_URL = os.getenv("DOCKER_REGISTRY_MIRROR_URL")

2
airbyte-ci/connectors/pipelines/pyproject.toml
View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "pipelines"
version = "4.1.4"
version = "4.2.0"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

7
airbyte-integrations/connectors/destination-bigquery/build.gradle
View File

@@ -14,6 +14,13 @@ airbyteJavaConnector {
useLocalCdk = false
}
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-this-escape"
}
}
application {
mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination'
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0',

6
airbyte-integrations/connectors/destination-iceberg/build.gradle
View File

@@ -52,12 +52,6 @@ dependencies {
testImplementation libs.testcontainers.postgresql
integrationTestJavaImplementation libs.testcontainers.postgresql
compileOnly 'org.projectlombok:lombok:1.18.24'
annotationProcessor 'org.projectlombok:lombok:1.18.24'
testCompileOnly 'org.projectlombok:lombok:1.18.24'
testAnnotationProcessor 'org.projectlombok:lombok:1.18.24'
testImplementation 'org.mockito:mockito-inline:4.7.0'
}

7
airbyte-integrations/connectors/destination-snowflake/build.gradle
View File

@@ -8,6 +8,13 @@ airbyteJavaConnector {
useLocalCdk = false
}
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-this-escape"
}
}
application {
mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestinationRunner'
// enable when profiling

2
airbyte-integrations/connectors/source-mongodb-v2/build.gradle
View File

@@ -1,6 +1,6 @@
plugins {
id 'airbyte-java-connector'
alias(libs.plugins.kotlin.jvm)
id 'org.jetbrains.kotlin.jvm' version '1.9.22'
}
airbyteJavaConnector {

4
airbyte-integrations/connectors/source-mssql/build.gradle
View File

@@ -9,9 +9,13 @@ airbyteJavaConnector {
}
java {
// TODO: rewrite code to avoid javac wornings in the first place
compileJava {
options.compilerArgs += "-Xlint:-try,-rawtypes"
}
compileTestFixturesJava {
options.compilerArgs += "-Xlint:-this-escape"
}
}
application {

69
build.gradle
View File

@@ -1,10 +1,10 @@
import com.github.spotbugs.snom.Confidence
import com.github.spotbugs.snom.Effort
import com.github.spotbugs.snom.SpotBugsTask
plugins {
id 'base'
id 'com.github.node-gradle.node' version '3.5.1'
id 'com.github.spotbugs' version '5.0.13'
id 'com.github.spotbugs' version '6.0.7'
id 'version-catalog'
id 'ru.vyarus.use-python'
}
@@ -181,12 +181,11 @@ subprojects { subproj ->
apply plugin: 'java'
apply plugin: 'java-test-fixtures'
apply plugin: 'jacoco'
apply plugin: 'com.github.spotbugs'
java {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
sourceCompatibility = JavaVersion.VERSION_21
targetCompatibility = JavaVersion.VERSION_21
compileJava {
options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing"]
}
@@ -218,50 +217,13 @@ subprojects { subproj ->
}
}
jacoco {
toolVersion = "0.8.8"
}
jacocoTestReport {
reports {
html.required = true
xml.required = true
csv.required = false
}
}
def jacocoTestReportTask = tasks.named('jacocoTestReport')
jacocoTestReportTask.configure {
dependsOn tasks.named('test')
}
jacocoTestCoverageVerification {
violationRules {
failOnViolation = false
rule {
element = 'CLASS'
excludes = ['**/*Test*', '**/generated*']
limit {
counter = 'BRANCH'
minimum = 0.8
}
limit {
counter = 'INSTRUCTION'
minimum = 0.8
}
}
}
}
spotbugs {
ignoreFailures = false
effort = 'max'
effort = Effort.valueOf(rootProject.ext.skipSlowTests ? 'MIN' : 'MAX')
excludeFilter.set rootProject.file('spotbugs-exclude-filter-file.xml')
reportLevel = 'high'
reportLevel = Confidence.valueOf('HIGH')
showProgress = false
toolVersion = '4.7.3'
if (rootProject.ext.skipSlowTests && isConnectorProject(subproj)) {
effort = 'min'
}
toolVersion = '4.8.3'
}
test {
@@ -278,6 +240,12 @@ subprojects { subproj ->
}
}
// This is required by mockito, see https://github.com/mockito/mockito/issues/3037.
jvmArgs "-XX:+EnableDynamicAgentLoading"
// This is also required, to prevent stderr spam starting with
// "OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader cl..."
jvmArgs "-Xshare:off"
// Set the timezone to UTC instead of picking up the host machine's timezone,
// which on a developer's laptop is more likely to be PST.
systemProperty 'user.timezone', 'UTC'
@@ -321,12 +289,6 @@ subprojects { subproj ->
exclude '**/io/airbyte/integrations/source/**'
exclude '**/io/airbyte/integrations/destination/**'
}
jacoco {
enabled = !rootProject.ext.skipSlowTests
excludes = ['**/*Test*', '**/generated*']
}
finalizedBy jacocoTestReportTask
}
dependencies {
@@ -348,8 +310,7 @@ subprojects { subproj ->
testFixturesImplementation libs.junit.pioneer
// adds owasp plugin
spotbugsPlugins libs.findsecbugs.plugin
implementation libs.spotbugs.annotations
implementation 'com.github.spotbugs:spotbugs-annotations:4.8.3'
}
tasks.withType(SpotBugsTask).configureEach {

1
buildSrc/src/main/groovy/airbyte-integration-test-java.gradle
View File

@@ -36,6 +36,7 @@ class AirbyteIntegrationTestJavaPlugin implements Plugin<Project> {
showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI")
}
jvmArgs = project.test.jvmArgs
systemProperties = project.test.systemProperties
maxParallelForks = project.test.maxParallelForks
maxHeapSize = project.test.maxHeapSize

4
deps.toml
View File

@@ -12,7 +12,7 @@ jooq = "3.13.4"
junit-jupiter = "5.9.1"
kotlin = "1.9.0"
log4j = "2.21.1"
lombok = "1.18.24"
lombok = "1.18.30"
postgresql = "42.6.0"
reactor = "3.5.2"
segment = "2.1.1"
@@ -87,7 +87,7 @@ log4j-over-slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j
log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j" }
lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" }
micrometer-statsd = { module = "io.micrometer:micrometer-registry-statsd", version = "1.9.3" }
mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "4.6.1" }
mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.10.0" }
mockk = { module = "io.mockk:mockk", version = "1.13.3" }
mongo-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "4.10.2" }
otel-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.14.0" }

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

6
spotbugs-exclude-filter-file.xml
View File

@@ -19,10 +19,4 @@
<Class name="io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore" />
<Bug code="SECOBDES" />
</Match>
<!-- Silenced during CDK migration. TODO: fix or replace this comment with description. -->
<Match>
<Class name="io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils" />
<Bug pattern="MS_SHOULD_BE_FINAL" />
</Match>
</FindBugsFilter>