kestra/.github/workflows/main.yml

name: Main

on:
  push:
    branches:
      - master
      - release
      - develop
    tags:
      - v*

  pull_request:
    branches:
      - master
      - release
      - develop
  repository_dispatch:
    types: [rebuild]
  workflow_dispatch:
    inputs:
      skip-test:
        description: 'Skip test'
        required: false
        type: string
        default: "false"

jobs:
  check:
    name: Check & Publish
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.x'
          architecture: 'x64'
      - uses: actions/setup-node@v3
        with:
          node-version: '14'
          check-latest: true

      # Services
      - name: Build the docker-compose stack
        run: docker-compose -f docker-compose-ci.yml up -d
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}

      # Caches
      - name: Gradle cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle*.properties') }}
          restore-keys: |
            ${{ runner.os }}-gradle-

      - name: Npm cache
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-npm-

      - name: Node cache
        uses: actions/cache@v3
        with:
          path: node
          key: ${{ runner.os }}-node-${{ hashFiles('ui/*.gradle') }}
          restore-keys: |
            ${{ runner.os }}-node-

      # JDK
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: 17

      - name: Validate Gradle wrapper
        uses: gradle/wrapper-validation-action@v1

      # Gradle check
      - name: Build with Gradle
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        env:
          GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
        run: |
          python3 -m pip install virtualenv
          echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json
          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
          ./gradlew check jacoco --parallel --no-daemon

      # Codecov
      - uses: codecov/codecov-action@v3
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      # Shadow Jar
      - name: Build jars
        run: ./gradlew executableJar --no-daemon

      # Upload artifacts
      - name: Upload jar
        uses: actions/upload-artifact@v3
        with:
          name: jar
          path: build/libs/

      - name: Upload Executable
        uses: actions/upload-artifact@v3
        with:
          name: exe
          path: build/executable/

      # GitHub Release
      - name: Create GitHub release
        uses: "marvinpinto/action-automatic-releases@latest"
        if: startsWith(github.ref, 'refs/tags/v')
        continue-on-error: true
        with:
          repo_token: "${{ secrets.GITHUB_TOKEN }}"
          prerelease: false
          files: |
            build/executable/*

  docker:
    name: Publish docker
    runs-on: ubuntu-latest
    needs: check
    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
    strategy:
      matrix:
        image:
          - name: ""
            plugins: ""
            packages: ""
          - name: "-full"
            plugins: io.kestra.storage:storage-azure:LATEST io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-airbyte:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-azure:LATEST io.kestra.plugin:plugin-powerbi:LATEST io.kestra.plugin:plugin-pulsar:LATEST io.kestra.plugin:plugin-cassandra:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-dbt:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-debezium-sqlserver:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fivetran:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-duckdb:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-pinot:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-rockset:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-trino:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-mqtt:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-servicenow:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-soda:LATEST io.kestra.plugin:plugin-spark:LATEST io.kestra.plugin:plugin-tika:LATEST
            packages: python3 python3-venv python-is-python3 nodejs curl zip unzip
    steps:
      - uses: actions/checkout@v3

      # Artifact
      - name: Download executable
        uses: actions/download-artifact@v3
        with:
          name: exe
          path: build/executable

      - name: Copy exe to image
        run: |
          cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra

      # Vars
      - name: Set image name
        id: vars
        run: |
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" ]]
          then
            echo "tag=latest" >> $GITHUB_OUTPUT
            echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          elif [[ $TAG == v* ]]
          then
            echo "tag=${TAG}" >> $GITHUB_OUTPUT
            echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          else
            echo "tag=${TAG}" >> $GITHUB_OUTPUT
            echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          fi

      # Docker setup
      - name: Set up QEMU
        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
        uses: docker/setup-buildx-action@v2

      # Docker Login
      - name: Login to DockerHub
        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      # Docker Build and push
      - name: Push to Docker Hub
        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
        uses: docker/build-push-action@v3
        with:
          context: .
          push: true
          tags: ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }}
          platforms: linux/amd64,linux/arm64
          build-args: |
            KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
            APT_PACKAGES=${{ matrix.image.packages }}

  maven:
    name: Publish to Maven
    runs-on: ubuntu-latest
    needs: check
    if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v3

      # Caches
      - name: Gradle cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle*.properties') }}
          restore-keys: |
            ${{ runner.os }}-gradle-

      - name: Npm cache
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-npm-

      - name: Node cache
        uses: actions/cache@v3
        with:
          path: node
          key: ${{ runner.os }}-node-${{ hashFiles('ui/*.gradle') }}
          restore-keys: |
            ${{ runner.os }}-node-

      # JDK
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: 17

      # Publish
      - name: Publish package to Sonatype
        if: github.ref == 'refs/heads/develop'
        env:
          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
        run: |
          mkdir -p ~/.gradle/
          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
          ./gradlew publishToSonatype --parallel --no-daemon

      # Release
      - name: Release package to Maven Central
        if: startsWith(github.ref, 'refs/tags/v')
        env:
          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
        run: |
          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
          ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository --no-daemon

  end:
    runs-on: ubuntu-latest
    needs:
      - check
      - maven
      - docker
    if: always()
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
    steps:
      # Slack
      - name: Slack notification
        uses: Gamesight/slack-workflow-status@master
        if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
          name: GitHub Actions
          icon_emoji: ':github-actions:'
          channel: 'C02DQ1A7JLR'