feat(docker): build a full image with all plugins

.dockerignore

@@ -0,0 +1,5 @@
+# Ignore everything
+**
+
+# Allow docker directories
+!docker/

.github/workflows/main.yml

@@ -39,11 +39,11 @@ jobs:
         with:
           node-version: '12'
           check-latest: true
+
       # Services
       - name: Build the docker-compose stack
         run: docker-compose -f docker-compose-ci.yml up -d
 
-      # Caches
       # Caches
       - name: Gradle cache
         uses: actions/cache@v2
@@ -119,7 +119,7 @@ jobs:
       # Slack
       - name: Slack notification
         uses: 8398a7/action-slack@v3
-        if: always()
+        if: failure()
         with:
           status: ${{ job.status }}
           job_name: Check
@@ -140,25 +140,62 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      # Artifact
       - name: Download executable
         uses: actions/download-artifact@v1
         with:
           name: exe
-      - name: Copy exe to image
-        run: cp exe/* docker/app/kestra && chmod +x docker/app/kestra
 
-      - name: Publish to Docker Hub
-        uses: elgohr/Publish-Docker-Github-Action@master
+      - name: Copy exe to image
+        run: |
+          cp exe/* docker/app/kestra && chmod +x docker/app/kestra
+
+      # Vars
+      - name: Set image name
+        id: vars
+        run: |
+          TAG=${GITHUB_REF#refs/*/}
+          if [[ $TAG = "master" ]]
+          then
+            echo ::set-output name=tag::latest
+          else
+            echo ::set-output name=tag::${TAG}
+          fi
+
+      # Docker
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
-          name: kestra/kestra
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-          tag_names: true
+
+      - name: Push to Docker Hub
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ format('kestra/kestra:{0}', steps.vars.outputs.tag) }}
+
+      - name: Push to Docker Hub Full
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ format('kestra/kestra:{0}-full', steps.vars.outputs.tag) }}
+          build-args: |
+            KESTRA_PLUGINS=org.kestra.storage.gcs:storage-gcs:LATEST org.kestra.storage.minio:storage-minio:LATEST org.kestra.task.aws:task-aws:LATEST org.kestra.task.gcp:task-gcp:LATEST org.kestra.task.kubernetes:task-kubernetes:LATEST org.kestra.task.serdes:task-serdes:LATEST org.kestra.task.notifications:task-notifications:LATEST org.kestra.task.crypto:task-crypto:LATEST org.kestra.task.jdbc:mysql:LATEST org.kestra.task.jdbc:postgres:LATEST
+            APT_PACKAGES=python3-pip python3-wheel python3-setuptools python3-virtualenv nodejs
 
       # Slack
       - name: Slack notification
         uses: 8398a7/action-slack@v3
-        if: failure()
+        if: always()
         with:
           status: ${{ job.status }}
           job_name: Publish docker

.gitignore

@@ -35,6 +35,10 @@ yarn.lock
 ### Docker
 /.env
 docker-compose*.overide.yml
+docker/app/plugins/*.jar
+docker/app/kestra
+docker/app/confs/*.yml
+docker/app/secrets/*.yml
 
 ### Build
 core/src/main/resources/gradle.properties

Dockerfile

@@ -1,7 +1,17 @@
 FROM openjdk:11-jre-slim
 
+ARG KESTRA_PLUGINS=""
+ARG APT_PACKAGES=""
+
 WORKDIR /app
 COPY docker /
+
+RUN if [ -n "${APT_PACKAGES}" ]; then apt-get update -y; apt-get install -y --no-install-recommends ${APT_PACKAGES}; apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/*; fi && \
+    if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS}; fi
+
+
 ENV MICRONAUT_CONFIG_FILES=/app/confs/application.yml
+
 ENTRYPOINT ["docker-entrypoint.sh"]
+
 CMD ["--help"]

docker-compose-dev.yml

@@ -1,84 +0,0 @@
-version: "3.6"
-
-volumes:
-  zookeeper-data:
-    driver: local
-  zookeeper-log:
-    driver: local
-  kafka-data:
-    driver: local
-  elasticsearch-data:
-    driver: local
-
-networks:
-  kestra: {}
-
-services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper
-    volumes:
-      - zookeeper-data:/var/lib/zookeeper/data
-      - zookeeper-log:/var/lib/zookeeper/log
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-    networks:
-      - kestra
-
-  kafka:
-    image: confluentinc/cp-kafka
-    volumes:
-      - kafka-data:/var/lib/kafka
-    environment:
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kestra:9092
-      KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
-    links:
-      - zookeeper
-    ports:
-      - 127.8.9.13:9092:9092
-    networks:
-      kestra:
-        aliases:
-        - kestra
-
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2
-    environment:
-      discovery.type: single-node
-      ES_JAVA_OPTS: "-Xms256m -Xmx256m"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    volumes:
-      - elasticsearch-data:/usr/share/elasticsearch/data
-    ports:
-      - 127.8.9.13:9200:9200
-    networks:
-      - kestra
-
-  kibana:
-    image: docker.elastic.co/kibana/kibana:7.4.2
-    environment:
-      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
-    ports:
-      - 127.8.9.13:5601:5601
-    networks:
-      - kestra
-
-  akhq:
-    image: tchiotludo/akhq
-    environment:
-      AKHQ_CONFIGURATION: |
-        kafkahq:
-          connections:
-            local:
-              properties:
-                bootstrap.servers: "kestra:9092"
-    ports:
-      - 127.8.9.13:28080:8080
-    networks:
-      - kestra

docker-compose.yml

@@ -9,6 +9,8 @@ volumes:
     driver: local
   elasticsearch-data:
     driver: local
+  kestra-data:
+    driver: local
 
 services:
   zookeeper:
@@ -46,40 +48,31 @@ services:
       - elasticsearch-data:/usr/share/elasticsearch/data
 
   kestra:
-    image: kestra/kestra:develop
+    image: kestra/kestra:develop-full
     command: server standalone
+    volumes:
+      - kestra-data:/app/storage
     environment:
-      MICRONAUT_APPLICATION_JSON: |
-        {
-          "kestra": {
-            "kafka": {
-              "client": {
-                "properties": {
-                  "bootstrap.servers": "kafka:9092"
-                }
-              }
-            },
-            "elasticsearch": {
-              "client": {
-                "http-hosts": "http://elasticsearch:9200"
-              }
-            },
-            "repository": {
-              "type": "elasticsearch"
-            },
-            "storage": {
-              "type": "local",
-              "local": {
-                "base-path": "/tmp/kestra"
-              }
-            },
-            "queue": {
-              "type": "kafka"
-            }
-          }
-        }
+      KESTRA_CONFIGURATION: |
+        kestra:
+          kafka:
+            client:
+              properties:
+                bootstrap.servers: kafka:9092
+          elasticsearch:
+            client:
+              http-hosts: http://elasticsearch:9200
+          repository:
+            type: elasticsearch
+          storage:
+            type: local
+            local:
+              base-path: "/app/storage"
+          queue:
+            type: kafka
+
     ports:
-      - 28080:8080
+      - 8080:8080
     links:
       - kafka
       - zookeeper

docker/usr/local/bin/docker-entrypoint.sh

@@ -2,8 +2,4 @@
 
 set -e
 
-if [ "${KESTRA_CONFIGURATION}" ]; then
-    echo "${KESTRA_CONFIGURATION}" > /app/confs/application.yml
-fi
-
 exec /app/kestra "$@"

gradle/jar/selfrun.bat

@@ -13,6 +13,14 @@ REM Plugins path default to pwd & must be exported as env var
 SET "current_dir=%~dp0"
 IF NOT DEFINED kestra_plugins_path (set "kestra_plugins_path=%current_dir%plugins\")
 
+REM Kestra configuration env vars
+
+IF NOT DEFINED kestra_configuration_path (set "kestra_configuration_path=%current_dir%confs\")
+
+IF DEFINED kestra_configuration (
+    echo %kestra_configuration% > "%kestra_configuration_path%application.yml"
+    set "micronaut_config_files=%kestra_configuration_path%application.yml"
+)
 
 REM Check java version
 FOR /f "delims=" %%w in ('java -fullversion 2^>^&1') do set java_fullversion=%%w

gradle/jar/selfrun.sh

@@ -2,6 +2,13 @@
 KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH:-"$(dirname "$0")/plugins"}
 export KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH}
 
+# Kestra configuration env vars
+KESTRA_CONFIGURATION_PATH=${KESTRA_CONFIGURATION_PATH:-"$(dirname "$0")/confs"}
+if [ "${KESTRA_CONFIGURATION}" ]; then
+    echo "${KESTRA_CONFIGURATION}" > "${KESTRA_CONFIGURATION_PATH}/application.yml"
+    export MICRONAUT_CONFIG_FILES="${KESTRA_CONFIGURATION_PATH}/application.yml"
+fi
+
 # Check java version
 JAVA_FULLVERSION=$(java -fullversion 2>&1)
 case "$JAVA_FULLVERSION" in