improve caching (#1005)

* python caching

* improve gradle caching

* store pip cache dir

* debugging info

* split build steps and temporarily remove testing infra from build

* fix formatting error

* fix another syntax error

* split repo caches

* switch pip dir

* use git hash instead of rehashing everything

* syntax

* list directories

* list after cache population

* list .docker

* specify custom output cache directory for docker

* only build bigquery and use buildx on ci

* add build command

* use docker driver instead of docker-container

* use correct key

* use local registry

* allow push failure

* make sure to tag first

* move part of node caching and restore rest of actual build

* cleanup

* clean up mssql gradle file

* toggle on inline cache

* try save and load method instead

* use separate tar for each image

* limit to 20

* always succeed even though disk space runs out

* try to just pull the public image to see if that's faster

* use cachefrom

* revert back to local registry version

* see how buildkit compares

* address review comments

* add comments

.github/workflows/gradle.yml

@@ -15,21 +15,44 @@ jobs:
       - name: Check images exist
         run: ./tools/bin/check_images_exist.sh
 
-      - name: Cache java deps
+      # docker should use restore-keys since these are just used for layer caching, not gradle task caching
+      - name: Docker Caching
         uses: actions/cache@v2
         with:
-          path: ~/.gradle
-          key: gradle-${{ hashFiles('**/*.gradle') }}
+          path: |
+            /tmp/docker-registry
+          key: docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-${{ github.sha }}
           restore-keys: |
-            gradle-
+            docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}-
+            docker-${{ runner.os }}-
 
-      - name: Cache node deps
+      - name: Pip Caching
         uses: actions/cache@v2
         with:
-          path: ~/.npm
-          key: node-${{ hashFiles('**/package-lock.json') }}
+          path: |
+            ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
           restore-keys: |
-            node-
+            pip-${{ runner.os }}-
+
+      - name: Npm Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.npm
+          key: npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            npm-${{ runner.os }}-
+
+      # this intentionally does not use restore-keys so we don't mess with gradle caching
+      - name: Gradle and Python Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            **/.venv
+          key: ${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
 
       - uses: actions/setup-java@v1
         with:
@@ -43,6 +66,20 @@ jobs:
         with:
           python-version: '3.7'
 
+      - name: Start local Docker registry
+        run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000
+
+      - name: Build
+        run: ./gradlew --no-daemon build --scan
+
+      - name: Ensure no file change
+        run: git status --porcelain && test -z "$(git status --porcelain)"
+
+      - name: Check documentation
+        if: success() && github.ref == 'refs/heads/master'
+        run: ./tools/site/link_checker.sh check_docs
+
+      # this should happen as late as possible so we don't accidentally have tests in the wrong stage depending on creds
       - name: Write Integration Test Credentials
         run: ./tools/bin/ci_credentials.sh
         env:
@@ -62,16 +99,6 @@ jobs:
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
           MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }}
 
-      - name: Build
-        run: ./gradlew --no-daemon build --scan
-
-      - name: Ensure no file change
-        run: git status --porcelain && test -z "$(git status --porcelain)"
-
-      - name: Check documentation
-        if: success() && github.ref == 'refs/heads/master'
-        run: ./tools/site/link_checker.sh check_docs
-
       - name: Run Integration Tests (PR)
         if: success() && github.ref != 'refs/heads/master'
         run: ./tools/bin/integration_test_pr.sh

airbyte-integrations/connectors/source-mssql/build.gradle

@@ -20,17 +20,15 @@ dependencies {
 
     // on rc version due to docker pull bug in current stable version.
     // issue: https://github.com/airbytehq/airbyte/issues/493
-    testCompile "org.testcontainers:mssqlserver:1.15.0-rc2"
+    testImplementation "org.testcontainers:mssqlserver:1.15.0-rc2"
 
     testImplementation 'org.apache.commons:commons-text:1.9'
     testImplementation 'org.apache.commons:commons-lang3:3.11'
     testImplementation 'org.apache.commons:commons-dbcp2:2.7.0'
-    testImplementation 'org.testcontainers:postgresql:1.15.0-rc2'
 
     testImplementation project(':airbyte-test-utils')
 
     integrationTestImplementation project(':airbyte-integrations:bases:standard-source-test')
-    testCompile "org.testcontainers:mssqlserver:1.15.0-rc2"
 
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
 }

buildSrc/src/main/groovy/airbyte-python.gradle

@@ -45,11 +45,17 @@ class AirbytePythonPlugin implements Plugin<Project> {
         project.task('installReqs', type: PythonTask) {
             module = "pip"
             command = "install -r requirements.txt"
+            inputs.file('requirements.txt')
+            outputs.file('build/installedreqs.txt')
+            outputs.cacheIf { true }
         }
 
         project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) {
             module = "pip"
             command = "install .[tests]"
+            inputs.file('setup.py')
+            outputs.file('build/installedtestreqs.txt')
+            outputs.cacheIf { true }
         }
 
         if(project.file('unit_tests').exists()) {

tools/bin/build_image.sh

@@ -5,7 +5,7 @@ set -e
 ROOT_DIR="$1"
 PROJECT_DIR="$2"
 DOCKERFILE="$3"
-TAG="$4"
+TAGGED_IMAGE="$4"
 ID_FILE="$5"
 
 cd "$ROOT_DIR"
@@ -14,4 +14,21 @@ assert_root
 
 cd "$PROJECT_DIR"
 
-DOCKER_BUILDKIT=1 docker build -f "$DOCKERFILE" . -t "$TAG" --iidfile "$ID_FILE"
+if [[ -z "$CI" ]]; then
+  # run standard build locally (not on CI)
+  DOCKER_BUILDKIT=1 docker build \
+    -f "$DOCKERFILE" . \
+    -t "$TAGGED_IMAGE" \
+    --iidfile "$ID_FILE"
+else
+  # run build with local docker registery for CI
+  docker pull localhost:5000/"$TAGGED_IMAGE" || true
+  DOCKER_BUILDKIT=1 docker build \
+    -f "$DOCKERFILE" . \
+    -t "$TAGGED_IMAGE" \
+    --iidfile "$ID_FILE" \
+    --cache-from localhost:5000/"$TAGGED_IMAGE" \
+    --build-arg BUILDKIT_INLINE_CACHE=1
+  docker tag "$TAGGED_IMAGE" localhost:5000/"$TAGGED_IMAGE"
+  docker push localhost:5000/"$TAGGED_IMAGE"
+fi