diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..cbda425c1c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +# Ignore everything +** + +# Allow docker directories +!docker/ \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd1542da97..46c1e6dfc5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,11 +39,11 @@ jobs: with: node-version: '12' check-latest: true + # Services - name: Build the docker-compose stack run: docker-compose -f docker-compose-ci.yml up -d - # Caches # Caches - name: Gradle cache uses: actions/cache@v2 @@ -119,7 +119,7 @@ jobs: # Slack - name: Slack notification uses: 8398a7/action-slack@v3 - if: always() + if: failure() with: status: ${{ job.status }} job_name: Check @@ -140,25 +140,62 @@ jobs: steps: - uses: actions/checkout@v2 + # Artifact - name: Download executable uses: actions/download-artifact@v1 with: name: exe - - name: Copy exe to image - run: cp exe/* docker/app/kestra && chmod +x docker/app/kestra - - name: Publish to Docker Hub - uses: elgohr/Publish-Docker-Github-Action@master + - name: Copy exe to image + run: | + cp exe/* docker/app/kestra && chmod +x docker/app/kestra + + # Vars + - name: Set image name + id: vars + run: | + TAG=${GITHUB_REF#refs/*/} + if [[ $TAG = "master" ]] + then + echo ::set-output name=tag::latest + else + echo ::set-output name=tag::${TAG} + fi + + # Docker + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 with: - name: kestra/kestra username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - tag_names: true + + - name: Push to Docker Hub + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ format('kestra/kestra:{0}', steps.vars.outputs.tag) }} + + - name: Push to Docker Hub Full + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ format('kestra/kestra:{0}-full', steps.vars.outputs.tag) }} + build-args: | + KESTRA_PLUGINS=org.kestra.storage.gcs:storage-gcs:LATEST org.kestra.storage.minio:storage-minio:LATEST org.kestra.task.aws:task-aws:LATEST org.kestra.task.gcp:task-gcp:LATEST org.kestra.task.kubernetes:task-kubernetes:LATEST org.kestra.task.serdes:task-serdes:LATEST org.kestra.task.notifications:task-notifications:LATEST org.kestra.task.crypto:task-crypto:LATEST org.kestra.task.jdbc:mysql:LATEST org.kestra.task.jdbc:postgres:LATEST + APT_PACKAGES=python3-pip python3-wheel python3-setuptools python3-virtualenv nodejs # Slack - name: Slack notification uses: 8398a7/action-slack@v3 - if: failure() + if: always() with: status: ${{ job.status }} job_name: Publish docker diff --git a/.gitignore b/.gitignore index 74d5ee8355..081d97bc36 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,10 @@ yarn.lock ### Docker /.env docker-compose*.overide.yml +docker/app/plugins/*.jar +docker/app/kestra +docker/app/confs/*.yml +docker/app/secrets/*.yml ### Build core/src/main/resources/gradle.properties \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9fa33dbca3..d0d03935b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,17 @@ FROM openjdk:11-jre-slim +ARG KESTRA_PLUGINS="" +ARG APT_PACKAGES="" + WORKDIR /app COPY docker / + +RUN if [ -n "${APT_PACKAGES}" ]; then apt-get update -y; apt-get install -y --no-install-recommends ${APT_PACKAGES}; apt-get clean && rm -rf /var/lib/apt/lists/* /var/tmp/*; fi && \ + if [ -n "${KESTRA_PLUGINS}" ]; then /app/kestra plugins install ${KESTRA_PLUGINS}; fi + + ENV MICRONAUT_CONFIG_FILES=/app/confs/application.yml + ENTRYPOINT ["docker-entrypoint.sh"] + CMD ["--help"] diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index 06f83d82cb..0000000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,84 +0,0 @@ -version: "3.6" - -volumes: - zookeeper-data: - driver: local - zookeeper-log: - driver: local - kafka-data: - driver: local - elasticsearch-data: - driver: local - -networks: - kestra: {} - -services: - zookeeper: - image: confluentinc/cp-zookeeper - volumes: - - zookeeper-data:/var/lib/zookeeper/data - - zookeeper-log:/var/lib/zookeeper/log - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - networks: - - kestra - - kafka: - image: confluentinc/cp-kafka - volumes: - - kafka-data:/var/lib/kafka - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kestra:9092 - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - links: - - zookeeper - ports: - - 127.8.9.13:9092:9092 - networks: - kestra: - aliases: - - kestra - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2 - environment: - discovery.type: single-node - ES_JAVA_OPTS: "-Xms256m -Xmx256m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - elasticsearch-data:/usr/share/elasticsearch/data - ports: - - 127.8.9.13:9200:9200 - networks: - - kestra - - kibana: - image: docker.elastic.co/kibana/kibana:7.4.2 - environment: - ELASTICSEARCH_HOSTS: http://elasticsearch:9200 - ports: - - 127.8.9.13:5601:5601 - networks: - - kestra - - akhq: - image: tchiotludo/akhq - environment: - AKHQ_CONFIGURATION: | - kafkahq: - connections: - local: - properties: - bootstrap.servers: "kestra:9092" - ports: - - 127.8.9.13:28080:8080 - networks: - - kestra diff --git a/docker-compose.yml b/docker-compose.yml index 39db297274..0c37610339 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ volumes: driver: local elasticsearch-data: driver: local + kestra-data: + driver: local services: zookeeper: @@ -46,40 +48,31 @@ services: - elasticsearch-data:/usr/share/elasticsearch/data kestra: - image: kestra/kestra:develop + image: kestra/kestra:develop-full command: server standalone + volumes: + - kestra-data:/app/storage environment: - MICRONAUT_APPLICATION_JSON: | - { - "kestra": { - "kafka": { - "client": { - "properties": { - "bootstrap.servers": "kafka:9092" - } - } - }, - "elasticsearch": { - "client": { - "http-hosts": "http://elasticsearch:9200" - } - }, - "repository": { - "type": "elasticsearch" - }, - "storage": { - "type": "local", - "local": { - "base-path": "/tmp/kestra" - } - }, - "queue": { - "type": "kafka" - } - } - } + KESTRA_CONFIGURATION: | + kestra: + kafka: + client: + properties: + bootstrap.servers: kafka:9092 + elasticsearch: + client: + http-hosts: http://elasticsearch:9200 + repository: + type: elasticsearch + storage: + type: local + local: + base-path: "/app/storage" + queue: + type: kafka + ports: - - 28080:8080 + - 8080:8080 links: - kafka - zookeeper diff --git a/docker/app/plugins/.gitkeep b/docker/app/plugins/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/usr/local/bin/docker-entrypoint.sh b/docker/usr/local/bin/docker-entrypoint.sh index 18d8f4ae15..7254b09067 100755 --- a/docker/usr/local/bin/docker-entrypoint.sh +++ b/docker/usr/local/bin/docker-entrypoint.sh @@ -2,8 +2,4 @@ set -e -if [ "${KESTRA_CONFIGURATION}" ]; then - echo "${KESTRA_CONFIGURATION}" > /app/confs/application.yml -fi - exec /app/kestra "$@" diff --git a/gradle/jar/selfrun.bat b/gradle/jar/selfrun.bat index 7fb1501a6a..881275adfe 100644 --- a/gradle/jar/selfrun.bat +++ b/gradle/jar/selfrun.bat @@ -13,6 +13,14 @@ REM Plugins path default to pwd & must be exported as env var SET "current_dir=%~dp0" IF NOT DEFINED kestra_plugins_path (set "kestra_plugins_path=%current_dir%plugins\") +REM Kestra configuration env vars + +IF NOT DEFINED kestra_configuration_path (set "kestra_configuration_path=%current_dir%confs\") + +IF DEFINED kestra_configuration ( + echo %kestra_configuration% > "%kestra_configuration_path%application.yml" + set "micronaut_config_files=%kestra_configuration_path%application.yml" +) REM Check java version FOR /f "delims=" %%w in ('java -fullversion 2^>^&1') do set java_fullversion=%%w diff --git a/gradle/jar/selfrun.sh b/gradle/jar/selfrun.sh index ee928a191c..f6e13776ea 100644 --- a/gradle/jar/selfrun.sh +++ b/gradle/jar/selfrun.sh @@ -2,6 +2,13 @@ KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH:-"$(dirname "$0")/plugins"} export KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH} +# Kestra configuration env vars +KESTRA_CONFIGURATION_PATH=${KESTRA_CONFIGURATION_PATH:-"$(dirname "$0")/confs"} +if [ "${KESTRA_CONFIGURATION}" ]; then + echo "${KESTRA_CONFIGURATION}" > "${KESTRA_CONFIGURATION_PATH}/application.yml" + export MICRONAUT_CONFIG_FILES="${KESTRA_CONFIGURATION_PATH}/application.yml" +fi + # Check java version JAVA_FULLVERSION=$(java -fullversion 2>&1) case "$JAVA_FULLVERSION" in