kestra/.github/workflows/main.yml

name: Main

on:
  push:
    branches:
      - develop
    tags:
      - v*
  pull_request:
    branches:
      - develop
  repository_dispatch:
    types: [ rebuild ]
  workflow_dispatch:
    inputs:
      skip-test:
        description: 'Skip test'
        type: choice
        required: true
        default: 'false'
        options:
          - "true"
          - "false"
      plugin-version:
        description: 'Plugin version'
        required: false
        type: string
        default: "LATEST"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
env:
  JAVA_VERSION: '21'
  DOCKER_APT_PACKAGES: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
  DOCKER_PYTHON_LIBRARIES: kestra
  PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }}
jobs:
  build-artifacts:
    name: Build Artifacts
    runs-on: ubuntu-latest
    timeout-minutes: 60
    outputs:
      docker-tag:  ${{ steps.vars.outputs.tag }}
      docker-artifact-name:  ${{ steps.vars.outputs.artifact }}
      plugins: ${{ steps.plugins-list.outputs.plugins }}
    steps:
      # Checkout
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # Checkout GitHub Actions
      - uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Setup build
      - uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          caches-enabled: true

      # Get Plugins List
      - name: Get Plugins List
        uses: ./.github/actions/plugins-list
        id: plugins-list
        with:
          plugin-version: ${{ env.PLUGIN_VERSION }}

      # Set Plugins List
      - name: Set Plugin List
        id: plugins
        run: |
          PLUGINS="${{ steps.plugins-list.outputs.plugins }}"
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" || $TAG == v* ]]; then
            echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT
          else
            echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT
          fi
      # Build
      - name: Build with Gradle
        run: |
          ./gradlew executableJar

      - name: Copy exe to image
        run: |
          cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra

      # Docker Tag
      - name: Set up Vars
        id: vars
        run: |
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" ]]
          then
            TAG="latest";
          elif [[ $TAG = "develop" ]]
          then
            TAG="develop";
          elif [[ $TAG = v* ]]
          then
            TAG="${TAG}";
          else
            TAG="build-${{ github.run_id }}";
          fi
          echo "tag=${TAG}" >> $GITHUB_OUTPUT
          echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT

      # Docker setup
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        uses: docker/setup-buildx-action@v3

      # Docker Build
      - name: Build & Export Docker Image
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        uses: docker/build-push-action@v6
        with:
          context: .
          push: false
          file: Dockerfile
          tags: |
            kestra/kestra:${{ steps.vars.outputs.tag }}
          build-args: |
            KESTRA_PLUGINS=${{ steps.plugins.outputs.plugins }}
            APT_PACKAGES=${{ env.DOCKER_APT_PACKAGES }}
            PYTHON_LIBRARIES=${{ env.DOCKER_PYTHON_LIBRARIES }}
          outputs: type=docker,dest=/tmp/${{ steps.vars.outputs.artifact }}.tar

      # Upload artifacts
      - name: Upload JAR
        uses: actions/upload-artifact@v4
        with:
          name: jar
          path: build/libs/

      - name: Upload Executable
        uses: actions/upload-artifact@v4
        with:
          name: exe
          path: build/executable/

      - name: Upload Docker
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.vars.outputs.artifact }}
          path: /tmp/${{ steps.vars.outputs.artifact }}.tar
  # Run Reusable Workflow from QA repository
  check-e2e:
    name: Check E2E Tests
    needs: build-artifacts
    if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
    uses: ./.github/workflows/e2e.yml
    strategy:
      fail-fast: false
      matrix:
        backends: ["postgres"]
    with:
      tags: oss
      docker-artifact-name: ${{ needs.build-artifacts.outputs.docker-artifact-name }}
      docker-image-tag: kestra/kestra:${{ needs.build-artifacts.outputs.docker-tag }}
      backend: ${{ matrix.backends }}
    secrets:
      GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
      GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
  check:
    env:
      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
      GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
    name: Check & Publish
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      # Checkout
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # Checkout GitHub Actions
      - uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Setup build
      - uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          python-enabled: true
          caches-enabled: true

      # Services
      - name: Build the docker-compose stack
        run: docker compose -f docker-compose-ci.yml up -d
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}

      # Gradle check
      - name: Build with Gradle
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        env:
          GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
        run: |
          echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json
          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
          ./gradlew check javadoc --parallel

      # Sonar
      - name: Analyze with Sonar
        if: ${{ env.SONAR_TOKEN != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        run: ./gradlew sonar --info

      # Allure check
      - name: Auth to Google Cloud
        id: auth
        if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }}
        uses: 'google-github-actions/auth@v2'
        with:
          credentials_json: '${{ secrets.GOOGLE_SERVICE_ACCOUNT }}'

      - uses: rlespinasse/github-slug-action@v4

      - name: Publish allure report
        uses: andrcuns/allure-publish-action@v2.7.1
        if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        env:
          GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          JAVA_HOME: /usr/lib/jvm/default-jvm/
        with:
          storageType: gcs
          resultsGlob: "**/build/allure-results"
          bucket: internal-kestra-host
          baseUrl: "https://internal.kestra.io"
          prefix: ${{ format('{0}/{1}/{2}', github.repository, env.GITHUB_HEAD_REF_SLUG != '' && env.GITHUB_HEAD_REF_SLUG || github.ref_name, 'allure/java') }}
          copyLatest: true
          ignoreMissingResults: true

      # Jacoco
      - name: 'Set up Cloud SDK'
        if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        uses: 'google-github-actions/setup-gcloud@v2'

      - name: 'Copy jacoco files'
        if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        run: |
          mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/
          mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/jacocoTestReport.xml
          gsutil -m rsync -d -r  build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}/{2}', github.repository, env.GITHUB_HEAD_REF_SLUG != '' && env.GITHUB_HEAD_REF_SLUG || github.ref_name, 'jacoco') }}

      # report test
      - name: Test Report
        uses: mikepenz/action-junit-report@v4
        if: success() || failure()
        with:
          report_paths: '**/build/test-results/**/TEST-*.xml'

      # Codecov
      - uses: codecov/codecov-action@v4
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  release:
    name: Github Release
    runs-on: ubuntu-latest
    needs: build-artifacts
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      # Download Exec
      - name: Download executable
        uses: actions/download-artifact@v4
        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: exe
          path: build/executable

      # GitHub Release
      - name: Create GitHub release
        uses: "marvinpinto/action-automatic-releases@latest"
        if: startsWith(github.ref, 'refs/tags/v')
        continue-on-error: true
        with:
          repo_token: "${{ secrets.GITHUB_TOKEN }}"
          prerelease: false
          files: |
            build/executable/*

  docker:
    name: Publish Docker
    runs-on: ubuntu-latest
    needs: [build-artifacts, check, check-e2e]
    if: github.ref == 'refs/heads/develop'
    strategy:
      matrix:
        image:
          - tag: ${{needs.build-artifacts.outputs.docker-tag}}-no-plugins
            packages: ""
            python-libraries: ""

          - tag: ${{needs.build-artifacts.outputs.docker-tag}}
            plugins: ${{needs.build-artifacts.outputs.plugins}}
            packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
            python-libraries: kestra
    steps:
      - uses: actions/checkout@v4
      # Docker setup
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # Docker Login
      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      # Vars
      - name: Set image name
        id: vars
        run: |
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" || $TAG == v* ]]; then
            echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          else
            echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          fi

      # Build Docker Image
      - name: Download executable
        uses: actions/download-artifact@v4
        with:
          name: exe
          path: build/executable

      - name: Copy exe to image
        run: |
          cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra

      # Docker Build and push
      - name: Build Docker Image
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          tags: kestra/kestra:${{ matrix.image.tag }}
          platforms: linux/amd64,linux/arm64
          build-args: |
            KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
            APT_PACKAGES=${{matrix.image.packages}}
            PYTHON_LIBRARIES=${{matrix.image.python-libraries}}

  maven:
    name: Publish to Maven
    runs-on: ubuntu-latest
    needs: [check]
    if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v4

      # Checkout GitHub Actions
      - uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Setup build
      - uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          caches-enabled: true

      # Publish
      - name: Publish package to Sonatype
        if: github.ref == 'refs/heads/develop'
        env:
          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
        run: |
          mkdir -p ~/.gradle/
          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
          ./gradlew publishToSonatype

      # Release
      - name: Release package to Maven Central
        if: startsWith(github.ref, 'refs/tags/v')
        env:
          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
        run: |
          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
          ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository
  end:
    runs-on: ubuntu-latest
    needs:
      - check-e2e
      - check
      - maven
      - docker
      - release
    if: always()
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
    steps:

      # Update
      - name: Update internal
        uses: benc-uk/workflow-dispatch@v1
        if: github.ref == 'refs/heads/develop'
        with:
          workflow: oss-build.yml
          repo: kestra-io/infra
          ref: master
          token: ${{ secrets.GH_PERSONAL_TOKEN }}

      # Slack
      - name: Slack notification
        uses: Gamesight/slack-workflow-status@master
        if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
          name: GitHub Actions
          icon_emoji: ':github-actions:'
          channel: 'C02DQ1A7JLR' # _int_git channel