gradle: build cache tweaks (#36055)

.github/workflows/gradle.yml

@@ -37,17 +37,18 @@ jobs:
             java:
               - '**/*.java'
               - '**/*.gradle'
+              - '**/*.kt'
               - 'airbyte-cdk/java/**/*'
 
   run-check:
     needs:
       - changes
     if: needs.changes.outputs.java == 'true'
-    # The gradle check task which we will run is embarrassingly parallelizable.
-    # We therefore run this on a machine with a maximum number of cores.
-    # We pay per time and per core, so there should be little difference in total cost.
-    # The latency overhead of setting up gradle prior to running the actual task adds up to about a minute.
-    runs-on: connector-test-xxlarge
+    # As of now, 16 cores seems to be enough.
+    # Any revision upwards should be based on a performance analysis of gradle scans.
+    # See https://github.com/airbytehq/airbyte/pull/36055 for an example of this,
+    # which explains why which we went down from 64 cores to 16.
+    runs-on: connector-test-large
     name: Gradle Check
     timeout-minutes: 30
     steps:
@@ -57,10 +58,6 @@ jobs:
         with:
           distribution: "zulu"
           java-version: "21"
-      - name: Install Pip
-        run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3
-      - name: Install Pyenv
-        run: python3 -m pip install virtualenv --user
       - name: Docker login
         # Some tests use testcontainers which pull images from DockerHub.
         uses: docker/login-action@v1

build.gradle

@@ -98,7 +98,7 @@ allprojects {
 
     spotbugs {
         ignoreFailures = false
-        effort = Effort.valueOf(System.getProperty('skipSlowTests', 'false') == 'false' ? 'MAX' : 'MIN')
+        effort = Effort.valueOf('MAX')
         excludeFilter.set rootProject.file('spotbugs-exclude-filter-file.xml')
         reportLevel = Confidence.valueOf('HIGH')
         showProgress = false
@@ -187,7 +187,6 @@ allprojects {
         def vJUnit = "5.10.2"
         def vJUnitJupiter = "5.11.0"
 
-
         testFixturesImplementation platform("org.junit:junit-bom:${vJUnit}")
         testFixturesImplementation "org.junit.jupiter:junit-jupiter-api:${vJUnit}"
         testFixturesImplementation "org.junit.jupiter:junit-jupiter-params:${vJUnit}"
@@ -209,6 +208,9 @@ allprojects {
     }
 
     tasks.withType(SpotBugsTask).configureEach {
+        if (name != "spotbugsMain" && System.getProperty('skipSlowTests', 'false') != 'false') {
+            enabled = false
+        }
         // Reports can be found under each subproject in build/spotbugs/
         reports {
             xml.required = false
@@ -216,5 +218,7 @@ allprojects {
         }
     }
 
-    javadoc.options.addStringOption('Xdoclint:none', '-quiet')
+    javadoc {
+        options.addStringOption('Xdoclint:none', '-quiet')
+    }
 }

settings.gradle

@@ -19,11 +19,10 @@ pluginManagement {
 // as much information as possible.
 plugins {
     id "com.gradle.enterprise" version "3.15.1"
-    id 'com.github.burrunan.s3-build-cache' version "1.5"
+    id 'com.github.burrunan.s3-build-cache' version "1.8.1"
 }
 
-ext.isCiServer = System.getenv().containsKey("CI")
-ext.isAirbyteCI = System.getenv().containsKey("RUN_IN_AIRBYTE_CI")
+final boolean isCI = System.getenv().containsKey("CI") || System.getenv().containsKey("RUN_IN_AIRBYTE_CI")
 
 dependencyResolutionManagement {
     // Set FAIL_ON_PROJECT_REPOS to ensure there are no more `repositories { ... }` blocks than necessary.
@@ -115,27 +114,30 @@ gradleEnterprise {
     buildScan {
         termsOfServiceUrl = "https://gradle.com/terms-of-service"
         termsOfServiceAgree = "yes"
-        uploadInBackground = (!isCiServer && !isAirbyteCI) // Disable in CI or scan URLs may not work.
+        uploadInBackground = !isCI // Disable in CI or scan URLs may not work.
         buildScanPublished { PublishedBuildScan scan ->
             file("scan-journal.log") << "${new Date()} - ${scan.buildScanId} - ${scan.buildScanUri}\n"
         }
     }
 }
 
-if (isCiServer || isAirbyteCI) {
+if (isCI) {
     buildCache {
         local {
             // Local build cache is dangerous as it might produce inconsistent results
             // in case developer modifies files while the build is running
-            enabled = isAirbyteCI
+            enabled = true
+            push = true
         }
         remote(com.github.burrunan.s3cache.AwsS3BuildCache) {
-            region = 'us-west-2' // close to dagger runners
+            enabled = !System.getenv().getOrDefault("S3_BUILD_CACHE_ACCESS_KEY_ID", "").isEmpty()
+            push = true
+            // The 'us-west-2' region was chosen because it was close to our v0.6 dagger runners
+            // we used to maintain in AWS, prior to migrating to v0.9 and to github runners.
+            // This may in fact no longer be the best choice, who knows? It's not terrible, at least.
+            region = 'us-west-2'
             bucket = 'ab-ci-cache'
             prefix = "${System.getProperty('s3BuildCachePrefix', 'connectors')}-ci-cache/"
-            push = isAirbyteCI
-            // Sometimes the env var is set, but with an empty value. Ignore this case.
-            enabled = !System.getenv().getOrDefault("S3_BUILD_CACHE_ACCESS_KEY_ID", "").isEmpty()
         }
     }
 }