CI: Add support for manifest-only connectors in the Connectors CI Matrix (Beta) (#59714)

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

.github/workflows/connector-test-command.yml

@@ -74,7 +74,7 @@ jobs:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.jvm-connectors-matrix) }}
       max-parallel: 5 # Limit number of parallel jobs
       fail-fast: false # Don't stop on first failure
-    name: Test ${{ matrix.connector }} Connector ${{ matrix.connector && '' || ' (no JVM-based connectors modified)'  }}
+    name: Test ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no JVM-based connectors modified)' || '' }}
     steps:
       - name: Checkout Airbyte
         if: matrix.connector
@@ -99,6 +99,15 @@ jobs:
           cache-read-only: false
           cache-write-only: false
 
+      # TODO: We can delete this step once Airbyte-CI is removed from Java integration tests.
+      - name: Set up Python (For Airbyte-CI)
+        if: matrix.connector
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          check-latest: true
+          update-environment: true
+
       - name: Install the latest version of uv
         if: matrix.connector
         uses: astral-sh/setup-uv@v6
@@ -111,27 +120,24 @@ jobs:
 
       - name: Install connector dependencies
         if: matrix.connector
-        run: |
-          poe connector ${{ matrix.connector }} install
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe install
 
       - name: Fetch connector secrets
         if: matrix.connector
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install pipx
-          uv tool run airbyte-cdk[dev] secrets fetch \
-            ${{ matrix.connector }} \
+          airbyte-cdk secrets fetch ${{ matrix.connector }} \
             --print-ci-secrets-masks
 
       - name: Run Unit Tests
         if: matrix.connector
-        run: |
-          poe connector ${{ matrix.connector }} test-unit-tests
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe test-unit-tests
 
       - name: Run Integration Tests
         if: matrix.connector
-        run: |
-          poe connector ${{ matrix.connector }} test-integration-tests
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe test-integration-tests
 
       - name: Publish Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -158,7 +164,7 @@ jobs:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.non-jvm-connectors-matrix) }}
       max-parallel: 5 # Limit number of parallel jobs
       fail-fast: false # Don't stop on first failure
-    name: Test ${{ matrix.connector }} Connector ${{ matrix.connector && '' || ' (no non-JVM-based connectors modified)'  }}
+    name: Test ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no non-JVM-based connectors modified)' || '' }}
     steps:
       - name: Checkout Airbyte
         if: matrix.connector
@@ -207,36 +213,26 @@ jobs:
           uv tool install pytest
           uv tool install mypy
 
-      - name: Get Connector Language
-        if: matrix.connector
-        id: get-language
-        continue-on-error: true # Will fail for now (null output) for manifest-only connectors
-        run: |
-          echo "LANGUAGE=$(poe -qq connector ${{ matrix.connector }} '-qq get-language')" >> $GITHUB_OUTPUT
-
       - name: Install connector dependencies
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} install
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe install
 
       - name: Fetch connector secrets
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
+        if: matrix.connector
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install pipx
-          pipx run airbyte-cdk[dev] secrets fetch \
-            ${{ matrix.connector }} \
+          airbyte-cdk secrets fetch ${{ matrix.connector }} \
             --print-ci-secrets-masks
 
       - name: Run Unit Tests
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} test-unit-tests
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe test-unit-tests
 
       - name: Run Integration Tests
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} test-integration-tests
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe test-integration-tests
 
       - name: Publish Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -263,7 +259,7 @@ jobs:
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }}
       max-parallel: 10 # Limit number of parallel jobs
-    name: Lint ${{ matrix.connector }} Connector ${{ matrix.connector && '' || ' (no connectors modified)'  }}
+    name: Lint ${{ matrix.connector }} Connector ${{ ! matrix.connector && ' (no connectors modified)' || '' }}
     steps:
       - name: Checkout Airbyte
         if: matrix.connector
@@ -325,27 +321,20 @@ jobs:
           uv tool install pytest
           uv tool install mypy
 
-      - name: Get Connector Language
-        if: matrix.connector
-        id: get-language
-        continue-on-error: true # Will fail for now (null output) for manifest-only connectors
-        run: |
-          echo "LANGUAGE=$(poe -qq connector ${{ matrix.connector }} '-qq get-language')" >> $GITHUB_OUTPUT
-
       - name: Install connector dependencies
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} install
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe install
 
       - name: Run format-check
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} format-check
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe format-check
 
       - name: Run lint check (info only)
-        if: ${{ matrix.connector && steps.get-language.outputs.LANGUAGE }}
-        run: |
-          poe connector ${{ matrix.connector }} lint-check
+        if: matrix.connector
+        working-directory: airbyte-integrations/connectors/${{ matrix.connector }}
+        run: poe lint-check
         # Most connectors can't pass lint checks, so this is non-blocking for now
         continue-on-error: true
 

airbyte-integrations/connectors/poe_tasks.toml

@@ -0,0 +1,5 @@
+# Any connector that doesn't have a `poe_tasks.toml` file will be assumed
+# to be a manifest-only connector. This is the most common connector type.
+include = [
+    "${POE_GIT_DIR}/poe-tasks/manifest-only-connector-tasks.toml",
+]

airbyte-integrations/connectors/source-klaviyo/poe_tasks.toml

@@ -1,3 +0,0 @@
-include = [
-    "${POE_GIT_DIR}/poe-tasks/manifest-only-connector-tasks.toml",
-]

poe-tasks/gradle-connector-tasks.toml

@@ -27,18 +27,24 @@
 
 [tasks]
 
-# Universal Connector Tasks
-# These tasks are available in all connectors, regardless of the language.
+get-language = "echo 'java'" # Use with -qq to get just the language name
+fetch-secrets = "airbyte-cdk secrets fetch"
+
+install = [
+  "gradle-warmup",
+  "install-cdk-cli",
+]
+
+gradle-warmup.help = "Warm up the Gradle daemon and cache by listing available tasks (no build)"
+gradle-warmup.cmd = "${POE_GIT_ROOT}/gradlew --no-daemon tasks --quiet"
+install-cdk-cli.help = "Install the CDK CLI for things like secrets and testing"
+install-cdk-cli.cmd = "uv tool install --upgrade 'airbyte-cdk[dev]'"
 
-install = "echo 'No install step for this connector.'"
 test-all = "gradle check"
 test-fast = "test-all"  # TODO: Add a fast-fail unit test definition
 format-check = "gradle check -x test"
 lint-check = "echo 'No lint check step for this connector.'"
 
-fetch-secrets = "uv tool run airbyte-cdk[dev] secrets fetch"
-get-language = "echo 'java'" # Use with -qq to get just the language name
-
 test-unit-tests = "gradle test"
 test-integration-tests.shell = '''
 if [ -d src/test-integration ]; then

poe-tasks/manifest-only-connector-tasks.toml

@@ -1,17 +1,55 @@
 # A shared set of tasks for Manifest-Only Airbyte connectors.
 #
-# This file should be included in connectors' `poe_tasks.toml` as follows:
-#
-# ```toml
-# `airbyte-integrations/connectors/<connector-name>/poe_tasks.toml`
-#
-# include = [
-#     "${POE_GIT_DIR}/poe-tasks/manifest-only-connector-tasks.toml",
-# ]
-# ```
+# Referenced from `airbyte-integrations/connectors/poe_tasks.toml` to apply to
+# all non-Java and non-Python connectors. By process of elimination, this covers
+# the manifest-only connectors.
+# 
+# If you find this tasks file is picked up by a connector that is *not* a
+# manifest-only connector, simply add a `poe_tasks.toml` file in the connector's
+# directory, and this will override the connectors' tasks to not use this file.
 
 [tasks]
 
-test-all = "uvx 'airbyte-cdk[dev]' connector test"
+get-language = "echo 'manifest-only'" # Use with -qq to get just the language name
+fetch-secrets = "airbyte-cdk secrets fetch ${POE_PWD}"
+
+install = [
+  "install-cdk-cli",
+  "install-unit-tests-project",
+]
+install-cdk-cli = "uv tool install --upgrade 'airbyte-cdk[dev]'"
+install-unit-tests-project.shell = '''
+if [ -f ${POE_PWD}/unit_tests/pyproject.toml ]; then
+  echo "Found 'unit_tests/pyproject.toml' file, installing unit tests..."
+  cd ${POE_PWD}/unit_tests
+  poetry install --no-root --all-extras
+else
+  echo "No unit tests defined; skipping unit tests installation."
+fi
+'''
+test-all = [
+  "test-unit-tests",
+  "test-integration-tests",
+]
+test-unit-tests.shell = '''
+if [ -f ${POE_PWD}/unit_tests/pyproject.toml ]; then
+  echo "Found 'unit_tests/pyproject.toml' file, running unit tests..."
+  cd ${POE_PWD}/unit_tests
+  poetry install --no-root --all-extras
+  poetry run pytest --junitxml=build/test-results/pytest-unit-tests-junit.xml
+else
+  echo "No unit tests defined; skipping unit tests."
+fi
+'''
+test-integration-tests = "airbyte-cdk connector test ${POE_PWD}"
 format-check = "echo 'No format check step for this connector.'"
-install = "echo 'No install step for this connector.'"
+
+lock.shell = '''
+if [ -f ${POE_PWD}/unit_tests/pyproject.toml ]; then
+  echo "Found 'unit_tests/pyproject.toml' file, locking unit tests project..."
+  cd ${POE_PWD}/unit_tests
+  poetry lock
+else
+  echo "No unit tests defined; skipping unit tests locking."
+fi
+'''

poe-tasks/poetry-connector-tasks.toml

@@ -16,10 +16,12 @@
 
 [tool.poe.tasks]
 
-# Universal Connector Tasks
-# These tasks are available in all connectors, regardless of the language.
+get-language = "echo 'python'" # Use with -qq to get just the language name
+fetch-secrets = "airbyte-cdk secrets fetch"
 
-install = "poetry install --all-extras"
+install = ["install-project", "install-cdk-cli"]
+install-project = "poetry install --all-extras"
+install-cdk-cli = "uv tool install --upgrade 'airbyte-cdk[dev]'"
 test-all = ["test-unit-tests", "test-integration-tests"]
 test-integration-tests = ["pytest-integration-tests"]
 test-unit-tests = ["pytest-unit-tests"]
@@ -30,9 +32,6 @@ lint-check = [
     "check-mypy",
 ]
 
-fetch-secrets = "uv tool run airbyte-cdk[dev] secrets fetch"
-get-language = "echo 'python'" # Use with -qq to get just the language name
-
 # Poetry Lock
 lock = "poetry lock"
 
@@ -47,7 +46,7 @@ fi
 '''
 
 pytest-integration-tests.shell = '''
-if [ -d integration_tests ]; then
+if ls integration_tests/test_*.py >/dev/null 2>&1; then
   poetry run pytest --junitxml=build/test-results/pytest-integration-tests-junit.xml integration_tests
 else
   echo "No 'integration_tests' directory found; skipping integration tests."

poe_tasks.toml

@@ -32,13 +32,17 @@ args = [
 ]
 
 [tasks.connector]
-help = "Run a poe task in a connector's directory. Usage: poe connector <connector-name> <task> [args...]"
+help = """Run a poe task in a connector's directory. Usage: poe connector <connector-name> <task> [args...]
+This does not yet work for manifest-only connectors, which need
+to be run from their own directory for now.
+"""
 
 shell = '''
 #!/bin/bash
 cd airbyte-integrations/connectors/${connector}
 poe ${task_and_args}
 '''
+cwd = "airbyte-integrations/connectors/${connector}"
 
 args = [
   { name = "connector", positional = true, help = "Name of the connector (e.g., source-hardcoded-records)" },