diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 0ba19c11bc5..bda5e2310a5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -89,11 +89,14 @@ jobs: - name: Generate Template scaffold run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan + # TODO(Issue-4915): Remove this once fix has been merged into upstream library. Rely on caching and only load credentials once. - name: Format - run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace + run: ./tools/bin/setup_gcs_creds.sh && GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace + env: + GOOGLE_CLOUD_STORAGE_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - name: Build - run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan + run: GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" @@ -220,14 +223,17 @@ jobs: - name: Install Pyenv run: python3 -m pip install virtualenv==16.7.9 --user + # TODO(Issue-4915): Remove this once fix has been merged into upstream library. Rely on caching and only load credentials once. - name: Format - run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace + run: ./tools/bin/setup_gcs_creds.sh && GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace + env: + GOOGLE_CLOUD_STORAGE_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" - name: Build - run: SUB_BUILD=PLATFORM ./gradlew build --scan + run: GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" && SUB_BUILD=PLATFORM ./gradlew build --scan - name: Ensure no file change run: git status --porcelain && test -z "$(git status --porcelain)" @@ -246,7 +252,7 @@ jobs: - name: Build Platform Docker Images if: success() && github.ref == 'refs/heads/master' - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + run: GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" ./gradlew composeBuild --scan env: GIT_REVISION: ${{ github.sha }} @@ -360,8 +366,11 @@ jobs: - name: Install Cypress Test Dependencies run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + # TODO(Issue-4915): Remove this once fix has been merged into upstream library. Rely on caching and only load credentials once. - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan + run: ./tools/bin/setup_gcs_creds.sh && GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan + env: + GOOGLE_CLOUD_STORAGE_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh @@ -455,8 +464,11 @@ jobs: HOME: /home/runner CHANGE_MINIKUBE_NONE_USER: true + # TODO(Issue-4915): Remove this once fix has been merged into upstream library. Rely on caching and only load credentials once. - name: Build Platform Docker Images - run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + run: ./tools/bin/setup_gcs_creds.sh && GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcs.json" SUB_BUILD=PLATFORM ./gradlew composeBuild --scan + env: + GOOGLE_CLOUD_STORAGE_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} - name: Run Logging Tests run: ./tools/bin/cloud_storage_logging_test.sh diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-commons/src/main/resources/log4j2.xml index a1f3a91d61f..84456bcbaf3 100644 --- a/airbyte-commons/src/main/resources/log4j2.xml +++ b/airbyte-commons/src/main/resources/log4j2.xml @@ -51,7 +51,7 @@ @@ -97,7 +97,7 @@ diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml index 2cdd5f01a67..40df43930d3 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - validate_output_from_all_streams: yes + empty_streams: [] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml index c52d0e3b37f..fc351f93d2d 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - validate_output_from_all_streams: yes + empty_streams: [] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" diff --git a/build.gradle b/build.gradle index 1ed691650a0..13401b32e79 100644 --- a/build.gradle +++ b/build.gradle @@ -152,6 +152,11 @@ subprojects { repositories { mavenCentral() + // TODO(Issue-4915): Remove this once fix has been merged into upstream library. + maven { + name 'GcsRepository' + url 'gcs://airbyte-test-public-bucket/' + } } pmd { @@ -213,7 +218,7 @@ subprojects { implementation 'org.slf4j:log4j-over-slf4j:1.7.30' // log4j1.2 bridge // Dependencies for logging to cloud storage, as well as the various clients used to do so. - implementation 'com.therealvan:appender-log4j2:3.1.0' + implementation 'com.therealvan:appender-log4j2:3.1.2-SNAPSHOT' implementation 'com.amazonaws:aws-java-sdk-s3:1.12.6' implementation 'com.google.cloud:google-cloud-storage:1.115.0' diff --git a/docs/deploying-airbyte/on-kubernetes.md b/docs/deploying-airbyte/on-kubernetes.md index 8c5627a5962..5ff1db85716 100644 --- a/docs/deploying-airbyte/on-kubernetes.md +++ b/docs/deploying-airbyte/on-kubernetes.md @@ -63,8 +63,7 @@ AWS_SECRET_ACCESS_KEY= # Endpoint where Minio is deployed at. S3_MINIO_ENDPOINT= ``` -The `S3_PATH_STYLE_ACCESS` should remain `true`. Although `S3_LOG_BUCKET_REGION` is used to create the Minio client, it's value is not actually used -and can remain untouched. +The `S3_PATH_STYLE_ACCESS` variable should remain `true`. The `S3_LOG_BUCKET_REGION` variable should remain empty. #### Configuring Custom S3 Log Location Replace the following variables in the `.env` file in the `kube/overlays/stable` directory: diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index ffaf8413e54..2966a14dc6f 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -41,7 +41,7 @@ LOG_LEVEL=INFO # S3/Minio Log Configuration S3_LOG_BUCKET=airbyte-dev-logs -S3_LOG_BUCKET_REGION=us-east-1 +S3_LOG_BUCKET_REGION= AWS_ACCESS_KEY_ID=minio AWS_SECRET_ACCESS_KEY=minio123 S3_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 diff --git a/tools/bin/setup_gcs_creds.sh b/tools/bin/setup_gcs_creds.sh new file mode 100755 index 00000000000..44a403c061e --- /dev/null +++ b/tools/bin/setup_gcs_creds.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# TODO(Issue-4915): Remove this hook as part of #4915. + +echo "$GOOGLE_CLOUD_STORAGE_CREDS" > "/tmp/gcs.json" +