Un-Revert OSS branch build for Cloud workflow (#11808)

* Revert "Revert "Build OSS branch for deploying to Cloud env (#11474)""

This reverts commit 55e3c1e051.

* add action to get dev branch tag to OSS project instead of doing it in cloud

* remove dev branch version action, going to do this in cloud afterall

.github/actions/build-and-push-branch/action.yml

@@ -0,0 +1,63 @@
+name: "Build OSS Branch and Push Minimum Required OSS Images"
+description: "Build jars and docker images tagged for a particular branch. Primarily used for running OSS branch code in Cloud."
+inputs:
+  branch_version_tag:
+    description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
+    required: false
+  dockerhub_token:
+    description: "Used to log in to dockerhub for pushing images"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: "Parse Input"
+      id: parse-input
+      shell: bash
+      run: |-
+        # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
+        #
+        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \
+          || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; }
+
+    - uses: actions/setup-java@v1
+      with:
+        java-version: "17"
+
+    - uses: actions/setup-node@v1
+      with:
+        node-version: "16.13.0"
+
+    - name: Set up CI Gradle Properties
+      run: |
+        mkdir -p ~/.gradle/
+        cat > ~/.gradle/gradle.properties <<EOF
+        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        org.gradle.workers.max=8
+        org.gradle.vfs.watch=false
+        EOF
+      shell: bash
+
+    - name: Build
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
+      shell: bash
+
+    - name: Publish to Maven Local
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew publishToMavenLocal
+      shell: bash
+
+    - name: Login to Docker (on Master)
+      uses: docker/login-action@v1
+      with:
+        username: airbytebot
+        password: ${{ inputs.dockerhub_token }}
+
+    - name: Push Docker Images
+      run: |
+        GIT_REVISION=$(git rev-parse HEAD)
+        [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+        VERSION=${{ steps.parse-input.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.build.yaml push
+      shell: bash

airbyte-bootloader/Dockerfile

@@ -1,10 +1,14 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-bootloader
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-bootloader/build.gradle

@@ -80,7 +80,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("bootloader",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyDocker)
 assemble.dependsOn(dockerBuildTask)

airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java

@@ -12,7 +12,7 @@ import java.util.Objects;
  */
 public class AirbyteVersion {
 
-  public static final String DEV_VERSION = "dev";
+  public static final String DEV_VERSION_PREFIX = "dev";
   public static final String AIRBYTE_VERSION_KEY_NAME = "airbyte_version";
 
   private final String version;
@@ -25,7 +25,7 @@ public class AirbyteVersion {
     this.version = version;
     final String[] parsedVersion = version.replace("\n", "").strip().split("-")[0].split("\\.");
 
-    if (version.equals(DEV_VERSION)) {
+    if (isDev()) {
       this.major = null;
       this.minor = null;
       this.patch = null;
@@ -66,7 +66,7 @@ public class AirbyteVersion {
    * Only the major and minor part of the Version is taken into account.
    */
   public int compatibleVersionCompareTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION))
+    if (isDev() || another.isDev())
       return 0;
     final int majorDiff = compareVersion(major, another.major);
     if (majorDiff != 0) {
@@ -100,7 +100,7 @@ public class AirbyteVersion {
    * Compares two Airbyte Version to check if they are equivalent (including patch version).
    */
   public int patchVersionCompareTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION)) {
+    if (isDev() || another.isDev()) {
       return 0;
     }
     final int majorDiff = compareVersion(major, another.major);
@@ -118,7 +118,7 @@ public class AirbyteVersion {
    * Compares two Airbyte Version to check if only the patch version was updated.
    */
   public boolean checkOnlyPatchVersionIsUpdatedComparedTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION)) {
+    if (isDev() || another.isDev()) {
       return false;
     }
     final int majorDiff = compareVersion(major, another.major);
@@ -133,7 +133,7 @@ public class AirbyteVersion {
   }
 
   public boolean isDev() {
-    return version.equals(DEV_VERSION);
+    return version.startsWith(DEV_VERSION_PREFIX);
   }
 
   /**

airbyte-config/init/build.gradle

@@ -19,6 +19,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("init",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("init", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyScripts)
 assemble.dependsOn(dockerBuildTask)

airbyte-container-orchestrator/Dockerfile

@@ -25,13 +25,16 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa
 RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
 RUN apt-get update && apt-get install -y kubectl
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-container-orchestrator
-ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"
+ENV VERSION=${VERSION}
+ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-${VERSION}/bin/${APPLICATION}"
 
 WORKDIR /app
 
 # Move orchestrator app
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-container-orchestrator/build.gradle

@@ -45,6 +45,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("container-orchestrator",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("container-orchestrator", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-db/lib/build.gradle

@@ -79,6 +79,6 @@ task copyInitSql(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("db",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("db", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyInitSql)
 assemble.dependsOn(dockerBuildTask)

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java

@@ -297,8 +297,8 @@ public class IntegrationRunner {
     final String version = parseConnectorVersion(env.getOrDefault("WORKER_CONNECTOR_IMAGE", ""));
     final String airbyteVersion = env.getOrDefault(EnvConfigs.AIRBYTE_VERSION, "");
     final String airbyteRole = env.getOrDefault(EnvConfigs.AIRBYTE_ROLE, "");
-    final boolean isDev = version.equals(AirbyteVersion.DEV_VERSION)
-        || airbyteVersion.equals(AirbyteVersion.DEV_VERSION)
+    final boolean isDev = version.startsWith(AirbyteVersion.DEV_VERSION_PREFIX)
+        || airbyteVersion.startsWith(AirbyteVersion.DEV_VERSION_PREFIX)
         || airbyteRole.equals("airbyter");
     if (isDev) {
       LOGGER.debug("Skip Sentry transaction for dev environment");

airbyte-metrics/reporter/Dockerfile

@@ -1,11 +1,15 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim AS metrics-reporter
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-metrics-reporter
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
+
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-metrics/reporter/build.gradle

@@ -25,6 +25,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("metrics-reporter",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-scheduler/app/Dockerfile

@@ -1,11 +1,14 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim AS scheduler
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-scheduler
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-scheduler/app/build.gradle

@@ -65,6 +65,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("scheduler",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("scheduler", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-server/Dockerfile

@@ -3,11 +3,14 @@ FROM openjdk:${JDK_VERSION}-slim AS server
 
 EXPOSE 8000
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-server
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-server/build.gradle

@@ -145,7 +145,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

airbyte-temporal/build.gradle

@@ -5,6 +5,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyScripts)
 assemble.dependsOn(dockerBuildTask)

airbyte-webapp/build.gradle

@@ -82,9 +82,9 @@ copyAssets.dependsOn npm_run_build
 assemble.dependsOn copyDocs
 copyDocker.dependsOn(npm_run_build)
 
-Task dockerBuildTask = getDockerBuildTask("webapp",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyBuild)
 dockerBuildTask.dependsOn(copyNginx)
 dockerBuildTask.dependsOn(copyDocs)
 dockerBuildTask.dependsOn(copyAssets)
-assemble.dependsOn(dockerBuildTask)
+assemble.dependsOn(dockerBuildTask)

airbyte-workers/Dockerfile

@@ -25,12 +25,15 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa
 RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
 RUN apt-get update && apt-get install -y kubectl
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-workers
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
 # Move worker app
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-workers/build.gradle

@@ -76,7 +76,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("worker",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

build.gradle

@@ -30,6 +30,15 @@ if (!env.containsKey('VERSION')) {
     throw new Exception('Version not specified in .env file...')
 }
 
+// `version` is used as the application build version for artifacts like jars
+// `image_tag` is used as the docker tag applied to built images.
+// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
+// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
+ext {
+    version = System.getenv("VERSION") ?: env.VERSION
+    image_tag = System.getenv("VERSION") ?: 'dev'
+}
+
 def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
     /*
     In java, we don't have a second linter/styling tool other than spotless so it doesn't really
@@ -129,9 +138,8 @@ spotless {
 check.dependsOn 'spotlessApply'
 
 @SuppressWarnings('GroovyAssignabilityCheck')
-def Task getDockerBuildTask(String artifactName, String projectDir) {
+def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
     return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
-        def buildTag = System.getenv('VERSION') ?: 'dev'
         def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
 
         def arch = System.getProperty("os.arch").toLowerCase()
@@ -149,6 +157,7 @@ def Task getDockerBuildTask(String artifactName, String projectDir) {
         buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
         buildArgs.put('ALPINE_IMAGE', alpineImage)
         buildArgs.put('POSTGRES_IMAGE', postgresImage)
+        buildArgs.put('VERSION', buildVersion)
     })
 }
 
@@ -170,7 +179,7 @@ allprojects {
     group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
     project.archivesBaseName = "${project.group}-${project.name}"
 
-    version = env.VERSION
+    version = rootProject.ext.version
 }
 
 // Java projects common configurations

docker-compose-cloud.build.yaml

@@ -0,0 +1,34 @@
+# Defines the minimum set of images needed to run Cloud.
+# Used to push OSS images that Cloud depends on.
+
+version: "3.7"
+
+services:
+  scheduler:
+    image: airbyte/scheduler:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-scheduler/app
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+  worker:
+    image: airbyte/worker:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-workers
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+  webapp:
+    image: airbyte/webapp:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-webapp
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}
+  metric-reporter:
+    image: airbyte/metrics-reporter:${VERSION}
+    build:
+      dockerfile: Dockerfile
+      context: airbyte-metrics/reporter
+      labels:
+        io.airbyte.git-revision: ${GIT_REVISION}

docs/contributing-to-airbyte/developing-locally.md

@@ -40,6 +40,12 @@ This will build all the code and run all the unit tests.
 
 `SUB_BUILD=PLATFORM ./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time.
 
+{% hint style="info" %}
+Optionally, you may pass a `VERSION` environment variable to the gradle build command. If present, gradle will use this value as a tag for all created artifacts (both Jars and Docker images).
+
+If unset, gradle will default to using the current VERSION in `.env` for Jars, and `dev` as the Docker image tag.
+{% endhint %}
+
 {% hint style="info" %}
 Gradle will use all CPU cores by default. If Gradle uses too much/too little CPU, tuning the number of CPU cores it uses to better suit a dev's need can help.
 

docs/contributing-to-airbyte/developing-on-docker.md

@@ -14,13 +14,13 @@ method takes 2 arguments:
 ## Adding a new docker build
 
 Once you have a `Dockerfile`, generating the docker image is done in the following way:
-- specify the artifact name and the project directory,
+- specify the artifact name, the project directory, the version, and the tag.
 - make sure that the Dockerfile is properly copied to the docker context dir before building the image
 - make the build docker task to depend on the `assemble` task.
 
 For example:
 ```groovy
-Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir)
+Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir, rootProject.ext.version, rootProject.ext.image_tag)
 dockerBuildTask.dependsOn(copyDocker)
 assemble.dependsOn(dockerBuildTask)
 ```
@@ -37,7 +37,7 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("init", project.projectDir)
+Task dockerBuildTask = getDockerBuildTask("init", project.projectDir, rootProject.ext.version, rootProject.ext.image_tag)
 dockerBuildTask.dependsOn(copyScripts)
 assemble.dependsOn(dockerBuildTask)
 ```