feat(cicd): add an executable jar

- Generate a self executable jar
- Refactor the docker image to use the executable
- Add a command `plugins install` to download a plugins
This commit is contained in:
tchiotludo
2020-01-03 17:37:49 +01:00
parent f4e9423ff4
commit 2fbae458c5
19 changed files with 476 additions and 106 deletions

View File

@@ -78,10 +78,18 @@ jobs:
# --parallel
# Shadow Jar
- name: Shadow jar
- name: Build jars
if: success() && matrix.java == '11'
run: ./gradlew shadowJar --no-daemon
run: ./gradlew executableJar --no-daemon
# Publish
- name: Publish package to Bintray
env:
BINTRAY_USER: ${{ secrets.BINTRAY_USER }}
BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }}
run: ./gradlew bintrayUpload --parallel --no-daemon
# Upload artifacts
- name: Upload jar
uses: actions/upload-artifact@v1
if: success() && matrix.java == '11'
@@ -89,6 +97,13 @@ jobs:
name: jar
path: build/libs/
- name: Upload Executable
uses: actions/upload-artifact@v1
if: success() && matrix.java == '11'
with:
name: exe
path: build/executable/
# Slack
- name: Slack notification
uses: 8398a7/action-slack@v2
@@ -102,55 +117,6 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
publish:
name: Publish package
runs-on: ubuntu-latest
needs: check
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop'
steps:
- uses: actions/checkout@v2
# Caches
- name: Gradle cache
uses: actions/cache@v1
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Gradle wrapper cache
uses: actions/cache@v1
with:
path: ~/.gradle/wrapper
key: ${{ runner.os }}-wrapper-${{ hashFiles('**/*.gradle') }}
restore-keys: |
${{ runner.os }}-wrapper-
# Java
- name: Set up JDK
uses: actions/setup-java@v1
with:
java-version: 11
- name: Publish package to GitHub
env:
BINTRAY_USER: ${{ secrets.BINTRAY_USER }}
BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }}
run: ./gradlew bintrayUpload --parallel --no-daemon
# Slack
- name: Slack notification
uses: 8398a7/action-slack@v2
if: failure()
with:
status: ${{ job.status }}
username: Github Actions
icon_emoji: ':octocat:'
channel: '#kestra'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
docker:
name: Publish docker
runs-on: ubuntu-latest
@@ -159,18 +125,20 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Download jar
- name: Download executable
uses: actions/download-artifact@v1
with:
name: jar
- name: Copy jar to image
run: cp jar/*.jar docker/app/libs/kestra.jar
name: exe
- name: Copy exe to image
run: cp exe/* docker/app/kestra && chmod +x docker/app/kestra
- name: Publish to Docker Hub
uses: elgohr/Publish-Docker-Github-Action@master
with:
name: kestra/kestra
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
tag_names: true
# Slack
- name: Slack notification

View File

@@ -75,11 +75,9 @@ allprojects {
runtime "ch.qos.logback:logback-classic:1.2.3"
runtime group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.12.1'
// utils
// lombok
annotationProcessor "org.projectlombok:lombok:" + lombokVersion
api group: 'com.google.guava', name: 'guava', version: '28.1-jre'
compileOnly 'org.projectlombok:lombok:' + lombokVersion
implementation 'com.github.jknack:handlebars:4.1.2'
// micronaut
annotationProcessor platform("io.micronaut:micronaut-bom:$micronautVersion")
@@ -99,6 +97,12 @@ allprojects {
// kestra
implementation group: 'com.devskiller.friendly-id', name: 'friendly-id', version: '1.1.0'
implementation 'com.github.jknack:handlebars:4.1.2'
// exposed utils
api group: 'com.google.guava', name: 'guava', version: '28.1-jre'
api group: 'commons-io', name: 'commons-io', version: '2.6'
api group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
}
}
@@ -169,12 +173,50 @@ jar {
}
shadowJar {
mergeServiceFiles()
dependsOn = [assembleFrontend]
archiveClassifier.set(null)
mergeServiceFiles()
}
shadowJar.dependsOn "assembleFrontend"
/**********************************************************************************************************************\
* Executable Jar
**********************************************************************************************************************/
def executableDir = file("${buildDir}/executable")
def executable = file("${buildDir}/executable/${project.name}-${project.version}")
task writeExecutableJar() {
group "build"
description "Write an executable jar from shadow jar"
dependsOn = [shadowJar]
doFirst {
executableDir.mkdirs()
}
doLast {
executable.write("")
executable.append("\n: <<END_OF_KESTRA_SELFRUN\r\n")
executable.append(file("gradle/jar/selfrun.bat").readBytes())
executable.append("\r\nEND_OF_KESTRA_SELFRUN\r\n\n")
executable.append(file("gradle/jar/selfrun.sh").readBytes())
executable.append(file("${buildDir}/libs/${project.name}-${project.version}.jar").readBytes())
executable.setExecutable(true)
}
}
task executableJar(type: Zip) {
group "build"
description "Zip the executable jar"
dependsOn = [writeExecutableJar]
archiveFileName = "${project.name}-${project.version}.zip"
destinationDirectory = file("${buildDir}/archives")
from executableDir
archiveClassifier.set(null)
}
/**********************************************************************************************************************\
* Jacoco
@@ -273,6 +315,7 @@ subprojects {
artifactId "kestra"
artifact shadowJar
artifact executableJar
} else {
from components.java
@@ -293,7 +336,7 @@ subprojects {
key = System.getenv('BINTRAY_KEY')
publications = ['BintrayMavenPublication']
publish = true
dryRun = false
dryRun = true
pkg {
userOrg = 'kestra'
name = project.name.contains('cli') ? "kestra" : project.name

View File

@@ -6,6 +6,20 @@ dependencies {
compile "io.micronaut:micronaut-http-client"
compile "io.micronaut:micronaut-http-server-netty"
// plugins
compile 'org.eclipse.aether:aether-api:1.1.0'
compile 'org.eclipse.aether:aether-spi:1.1.0'
compile 'org.eclipse.aether:aether-util:1.1.0'
compile 'org.eclipse.aether:aether-impl:1.1.0'
compile 'org.eclipse.aether:aether-connector-basic:1.1.0'
compile 'org.eclipse.aether:aether-transport-file:1.1.0'
compile 'org.eclipse.aether:aether-transport-http:1.1.0'
compile('org.apache.maven:maven-aether-provider:3.1.0') {
// sisu dependency injector is not used
exclude group: 'org.eclipse.sisu'
}
// modules
compile project(":core")

View File

@@ -1,6 +1,7 @@
package org.kestra.cli;
import io.micronaut.configuration.picocli.PicocliRunner;
import org.kestra.cli.commands.plugins.PluginCommand;
import org.kestra.cli.commands.servers.StandAloneCommand;
import org.kestra.cli.commands.TestCommand;
import org.kestra.cli.commands.servers.WebServerCommand;
@@ -23,6 +24,7 @@ import java.util.concurrent.Callable;
TestCommand.class,
WebServerCommand.class,
WorkerCommand.class,
PluginCommand.class
}
)
public class App implements Callable<Object> {

View File

@@ -0,0 +1,19 @@
package org.kestra.cli.commands.plugins;
import lombok.extern.slf4j.Slf4j;
import org.kestra.cli.AbstractCommand;
import picocli.CommandLine;
@CommandLine.Command(
name = "plugins",
description = "handle plugins",
subcommands = {
PluginInstallCommand.class
}
)
@Slf4j
public class PluginCommand extends AbstractCommand {
public PluginCommand() {
super(false);
}
}

View File

@@ -0,0 +1,68 @@
package org.kestra.cli.commands.plugins;
import io.micronaut.context.annotation.Value;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.eclipse.aether.resolution.DependencyResolutionException;
import org.kestra.cli.AbstractCommand;
import org.kestra.cli.plugins.PluginDownloader;
import picocli.CommandLine;
import javax.annotation.Nullable;
import javax.inject.Inject;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
@CommandLine.Command(
name = "install",
description = "install a plugin"
)
@Slf4j
public class PluginInstallCommand extends AbstractCommand {
@CommandLine.Parameters(index = "0..*", description = "the plugins to install")
List<String> dependencies = new ArrayList<>();
@Inject
PluginDownloader pluginDownloader;
@Value("${kestra.plugins.path}")
Path pluginsPath;
public PluginInstallCommand() {
super(false);
}
@SuppressWarnings("ResultOfMethodCallIgnored")
@Override
public void run() {
super.run();
if (!pluginsPath.toFile().exists()) {
pluginsPath.toFile().mkdir();
}
try {
List<URL> resolveUrl = pluginDownloader.resolve(dependencies);
log.debug("Resolved Plugin(s) with {}", resolveUrl);
for (URL url: resolveUrl) {
Files.copy(
Paths.get(url.toURI()),
Paths.get(pluginsPath.toString(), FilenameUtils.getName(url.toString())),
StandardCopyOption.REPLACE_EXISTING
);
}
log.info("Successfully installed plugins {} into {}", dependencies, pluginsPath);
} catch (DependencyResolutionException | URISyntaxException | IOException e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -0,0 +1,138 @@
package org.kestra.cli.plugins;
import com.google.common.collect.ImmutableList;
import io.micronaut.context.annotation.Value;
import lombok.extern.slf4j.Slf4j;
import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
import org.eclipse.aether.DefaultRepositorySystemSession;
import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
import org.eclipse.aether.artifact.DefaultArtifact;
import org.eclipse.aether.collection.CollectRequest;
import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
import org.eclipse.aether.graph.Dependency;
import org.eclipse.aether.graph.DependencyFilter;
import org.eclipse.aether.impl.DefaultServiceLocator;
import org.eclipse.aether.repository.LocalRepository;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.resolution.DependencyRequest;
import org.eclipse.aether.resolution.DependencyResolutionException;
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
import org.eclipse.aether.spi.connector.transport.TransporterFactory;
import org.eclipse.aether.transport.file.FileTransporterFactory;
import org.eclipse.aether.transport.http.HttpTransporterFactory;
import org.eclipse.aether.util.filter.DependencyFilterUtils;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.util.List;
import java.util.stream.Collectors;
@Singleton
@Slf4j
public class PluginDownloader {
private List<RepositoryConfig> repositoryConfigs;
private final RepositorySystem system;
private final RepositorySystemSession session;
@Inject
public PluginDownloader(
List<RepositoryConfig> repositoryConfigs,
@Nullable @Value("${kestra.plugins.local-repository-path}") String localRepositoryPath
) {
this.repositoryConfigs = repositoryConfigs;
this.system = repositorySystem();
this.session = repositorySystemSession(system, localRepositoryPath);
}
public List<URL> resolve(List<String> dependencies) throws DependencyResolutionException, MalformedURLException {
List<RemoteRepository> repositories = remoteRepositories();
ImmutableList.Builder<URL> urls = ImmutableList.builder();
for (String dependency : dependencies) {
log.debug("Resolving plugin {}", dependency);
List<ArtifactResult> artifactResults = resolveArtifacts(repositories, dependency);
List<URL> localUrls = resolveUrls(artifactResults);
log.debug("Resolved Plugin {} with {}", dependency, localUrls);
urls.addAll(localUrls);
}
return urls.build();
}
private List<RemoteRepository> remoteRepositories() {
return repositoryConfigs
.stream()
.map(repositoryConfig ->
new RemoteRepository.Builder(
repositoryConfig.getId(),
repositoryConfig.getType(),
repositoryConfig.getUrl()
)
.build()
)
.collect(Collectors.toList());
}
private static RepositorySystem repositorySystem() {
DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
locator.addService(TransporterFactory.class, FileTransporterFactory.class);
locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
return locator.getService(RepositorySystem.class);
}
private RepositorySystemSession repositorySystemSession(RepositorySystem system, String localRepositoryPath) {
DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
if (localRepositoryPath == null) {
try {
localRepositoryPath = Files.createTempDirectory(this.getClass().getSimpleName().toLowerCase())
.toAbsolutePath()
.toString();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
LocalRepository localRepo = new LocalRepository(localRepositoryPath);
session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
return session;
}
private List<ArtifactResult> resolveArtifacts(List<RemoteRepository> repositories, String dependency) throws DependencyResolutionException {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter("jar");
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(artifact, "jar"));
collectRequest.setRepositories(repositories);
DependencyRequest depRequest = new DependencyRequest(collectRequest, classpathFlter);
return system.resolveDependencies(session, depRequest).getArtifactResults();
}
private List<URL> resolveUrls(List<ArtifactResult> artifactResults) throws MalformedURLException {
ImmutableList.Builder<URL> urls = ImmutableList.builder();
for (ArtifactResult artifactResult : artifactResults) {
URL url;
url = artifactResult.getArtifact().getFile().toPath().toUri().toURL();
urls.add(url);
}
return urls.build();
}
}

View File

@@ -0,0 +1,19 @@
package org.kestra.cli.plugins;
import io.micronaut.context.annotation.EachProperty;
import io.micronaut.context.annotation.Parameter;
import lombok.Getter;
@EachProperty("kestra.plugins.repositories")
@Getter
public class RepositoryConfig {
String id;
String type = "default";
String url;
public RepositoryConfig(@Parameter String id) {
this.id = id;
}
}

View File

@@ -200,3 +200,12 @@ kestra:
}
}
}
plugins:
repositories:
central:
url: https://repo.maven.apache.org/maven2/
jcenter:
url: https://jcenter.bintray.com/
kestra:
url: https://dl.bintray.com/kestra/maven

View File

@@ -9,4 +9,4 @@ class TodoTest {
void todo() {
assertTrue(true);
}
}
}

View File

@@ -0,0 +1,34 @@
package org.kestra.cli.commands.plugins;
import io.micronaut.test.annotation.MicronautTest;
import org.junit.jupiter.api.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
@MicronautTest
class PluginInstallCommandTest {
@Inject
PluginInstallCommand pluginInstallCommand;
@Test
void run() throws IOException {
pluginInstallCommand.pluginsPath = Files.createTempDirectory(PluginInstallCommandTest.class.getSimpleName());
pluginInstallCommand.dependencies = Collections.singletonList("org.kestra.task.notifications:task-notifications:0.1.0");
pluginInstallCommand.run();
List<Path> files = Files.list(pluginInstallCommand.pluginsPath).collect(Collectors.toList());
assertThat(files.size(), is(1));
assertThat(files.get(0).getFileName().toString(), is("task-notifications-0.1.0.jar"));
}
}

View File

@@ -0,0 +1,8 @@
kestra:
plugins:
path: /tmp/plugins
repositories:
central:
url: https://repo.maven.apache.org/maven2/
kestra:
url: https://dl.bintray.com/kestra/maven

View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!-- Remove logback startup log -->
<statusListener class="ch.qos.logback.core.status.NopStatusListener" />
<property name="pattern" value="%d{ISO8601} %highlight(%-5.5level) %magenta(%-12.12thread) %cyan(%-12.12logger{12}) %msg%n" />
<withJansi>true</withJansi>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<target>System.out</target>
<immediateFlush>true</immediateFlush>
<encoder>
<pattern>${pattern}</pattern>
</encoder>
</appender>
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
<logger name="org.kestra" level="INFO" />
<logger name="flow" level="INFO" />
<logger name="org.kestra.runner.kafka.services" level="WARN" />
<!-- The configuration '%s' was supplied but isn't a known config. > https://github.com/apache/kafka/pull/5876 -->
<logger name="org.apache.kafka.clients.producer.ProducerConfig" level="ERROR" />
<logger name="org.apache.kafka.clients.admin.AdminClientConfig" level="ERROR" />
<logger name="org.apache.kafka.clients.consumer.ConsumerConfig" level="ERROR" />
<!--- Error registering AppInfo mbean -->
<logger name="org.apache.kafka.common.utils.AppInfoParser" level="ERROR" />
</configuration>

View File

@@ -1,34 +0,0 @@
###########################################################################
# jvm.options #
# #
# - all flags defined here will be used to startup the JVM #
# - one flag should be specified per line #
# - lines that do not start with '-' will be ignored #
# - only static flags are accepted (no variables or parameters) #
# - dynamic flags will be appended to these on cassandra-env #
###########################################################################
# Server Hotspot JVM
-server
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# set to headless, just in case
-Djava.awt.headless=true
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/tmp/heapdump.log
# Do not rely on the system configuration
-Dfile.encoding=UTF-8
-Duser.timezone=UTC
# Jmx Remote
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port=8686
-Dcom.sun.management.jmxremote.local.only=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.ssl=false

View File

@@ -1,10 +0,0 @@
#!/usr/bin/env sh
# Read user-defined JVM options from jvm.options file
JVM_OPTS_FILE=${JVM_OPTS_FILE:-/app/jvm.options}
for JVM_OPT in `grep "^-" ${JVM_OPTS_FILE}`
do
JAVA_OPTS="${JAVA_OPTS} ${JVM_OPT}"
done
/usr/local/openjdk-11/bin/java ${JAVA_OPTS} -cp /app/libs/kestra.jar:/app/libs/plugins/* org.kestra.cli.App "$@"

42
gradle/jar/selfrun.bat Normal file
View File

@@ -0,0 +1,42 @@
@echo off
REM NOTE: Do not use backquotes in this bat file because backquotes are unintentionally recognized by sh.
REM NOTE: Just quotes are available for [ for /f "delims=" %%w ('...') ].
setlocal
REM Do not use %0 to identify the JAR (bat) file.
REM %0 is just "kestra" when run by just "> kestra" while %0 is "kestra.bat" when run by "> kestra.bat".
SET this=%~f0
REM Plugins path default to pwd & must be exported as env var
SET "current_dir=%~dp0"
IF NOT DEFINED kestra_plugins_path (set "kestra_plugins_path=%current_dir%plugins\")
REM Check java version
FOR /f "delims=" %%w in ('java -fullversion 2^>^&1') do set java_fullversion=%%w
ECHO %java_fullversion% | find " full version ""1." > NUL
IF NOT ERRORLEVEL 1 (set java_version=1)
ECHO %java_fullversion% | find " full version ""9" > NUL
IF NOT ERRORLEVEL 1 (set java_version=9)
ECHO %java_fullversion% | find " full version ""10" > NUL
IF NOT ERRORLEVEL 1 (set java_version=10)
IF NOT DEFINED java_version (set java_version=0)
IF %java_version% NEQ 0 (
ECHO [ERROR] Kestra require at least Java 11.. 1>&2
EXIT 1
)
echo java %JAVA_OPTS% -cp "%this%:%kestra_plugins_path%*" org.kestra.cli.App %*
java %JAVA_OPTS% -cp "%this%;%kestra_plugins_path%*" org.kestra.cli.App %*
ENDLOCAL
exit /b %ERRORLEVEL%

16
gradle/jar/selfrun.sh Normal file
View File

@@ -0,0 +1,16 @@
# Plugins path default to pwd & must be exported as env var
KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH:-"$(dirname "$0")/plugins"}
export KESTRA_PLUGINS_PATH=${KESTRA_PLUGINS_PATH}
# Check java version
JAVA_FULLVERSION=$(java -fullversion 2>&1)
case "$JAVA_FULLVERSION" in
[a-z]*\ full\ version\ \"\(1|9|10\)\..*\")
echo "[ERROR] Kestra require at least Java 11." 1>&2
exit 1
;;
esac
# Exec
exec java ${JAVA_OPTS} -cp "$0:${KESTRA_PLUGINS_PATH}/*" org.kestra.cli.App "$@"
exit 127