jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: correct regression test report path for LLM evaluation (#69796)

Browse Source 
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Aaron ("AJ") Steers
2025-11-20 17:46:11 -08:00
committed by GitHub
parent cf1379d105
commit d140a98dab
1 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
.github/workflows/run-regression-tests-command.yml vendored
View File

@@ -218,12 +218,26 @@ jobs:
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} ${{ inputs.connector_filter }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url="https://github.com/airbytehq/airbyte/pull/${{ github.event.inputs.pr }}" ${{ env.READ_WITH_STATE_FLAG }} ${{ env.DISABLE_PROXY_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} ${{ env.CONTROL_VERSION }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'
- name: Upload regression test report
- name: Locate regression test report
if: always() && github.event_name == 'workflow_dispatch'
id: locate-report
run: |
# Find the most recent report.html file in /tmp/live_tests_artifacts/
REPORT_PATH=$(find /tmp/live_tests_artifacts -name "report.html" -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | tail -1 | cut -f2- -d" ")
if [ -n "$REPORT_PATH" ]; then
echo "report_path=$REPORT_PATH" >> "$GITHUB_OUTPUT"
echo "Found report at: $REPORT_PATH"
else
echo "report_path=" >> "$GITHUB_OUTPUT"
echo "No report.html found in /tmp/live_tests_artifacts/"
fi
- name: Upload regression test report
if: always() && github.event_name == 'workflow_dispatch' && steps.locate-report.outputs.report_path != ''
uses: actions/upload-artifact@v4
with:
name: regression-test-report
path: /tmp/regression_tests_artifacts/report.html
path: ${{ steps.locate-report.outputs.report_path }}
if-no-files-found: ignore
- name: Append regression outcome
@@ -235,7 +249,7 @@ jobs:
edit-mode: append
body: |
> Regression tests: ${{ steps.run-regression-tests.outcome == 'success' && '✅ PASSED' || steps.run-regression-tests.outcome == 'failure' && '❌ FAILED' || steps.run-regression-tests.outcome == 'cancelled' && '⚠️ CANCELLED' || steps.run-regression-tests.outcome == 'skipped' && '⏭️ SKIPPED' || '❓ UNKNOWN' }}
> Report: ${{ hashFiles('/tmp/regression_tests_artifacts/report.html') != '' && 'artifact `regression-test-report` available in the run' || 'not generated' }}
> Report: ${{ steps.locate-report.outputs.report_path != '' && 'artifact `regression-test-report` available in the run' || 'not generated' }}
- name: Install live-tests dependencies for LLM evaluation
if: always() && github.event_name == 'workflow_dispatch'
@@ -252,7 +266,7 @@ jobs:
echo "Ollama server started and model pulled"
- name: Evaluate Regression Test Report with LLM
if: always() && github.event_name == 'workflow_dispatch'
if: always() && github.event_name == 'workflow_dispatch' && steps.locate-report.outputs.report_path != ''
id: llm-eval
continue-on-error: true
working-directory: airbyte-ci/connectors/live-tests
@@ -265,18 +279,16 @@ jobs:
echo "ran=false" >> "$GITHUB_OUTPUT"
echo "result=error" >> "$GITHUB_OUTPUT"
# Find the most recent report.html file in /tmp/regression_tests_artifacts/
REPORT_PATH=$(find /tmp/regression_tests_artifacts -name "report.html" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" ")
REPORT_PATH="${{ steps.locate-report.outputs.report_path }}"
if [ -z "$REPORT_PATH" ]; then
echo "Error: No report.html found in /tmp/regression_tests_artifacts/" >&2
echo "Error: No report path provided from locate-report step" >&2
echo "## ⚠️ LLM Evaluation Skipped" >> "$GITHUB_STEP_SUMMARY"
echo "No regression test report found. The tests may have failed to generate a report." >> "$GITHUB_STEP_SUMMARY"
exit 1
fi
echo "Found report at: $REPORT_PATH"
echo "Running LLM evaluation..."
echo "Evaluating report at: $REPORT_PATH"
# Run the evaluation script
OUT_JSON="$RUNNER_TEMP/llm_eval.json"