Add Filesystem based standard test base (#1333)

airbyte-integrations/bases/base-standard-source-test-file/.dockerignore

@@ -0,0 +1,4 @@
+*
+!Dockerfile
+!build
+!entrypoint.sh

airbyte-integrations/bases/base-standard-source-test-file/Dockerfile

@@ -0,0 +1,30 @@
+FROM openjdk:14.0.2-slim
+
+# Install Docker to launch worker images. Eventually should be replaced with Docker-java.
+# See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java
+RUN apt-get update && apt-get install -y \
+                          apt-transport-https \
+                          ca-certificates \
+                          curl \
+                          gnupg-agent \
+                          software-properties-common
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+RUN add-apt-repository \
+       "deb [arch=amd64] https://download.docker.com/linux/debian \
+       $(lsb_release -cs) \
+       stable"
+RUN apt-get update && apt-get install -y docker-ce-cli jq
+
+ENV APPLICATION base-standard-source-test-file
+
+WORKDIR /app
+
+COPY entrypoint.sh .
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+ENTRYPOINT ["/app/entrypoint.sh"]
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/base-standard-source-test-file

airbyte-integrations/bases/base-standard-source-test-file/build.gradle

@@ -0,0 +1,17 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+}
+
+dependencies {
+    implementation project(':airbyte-config:models')
+    implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-workers')
+    implementation project(':airbyte-integrations:bases:standard-source-test')
+
+    implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.standardtest.source.fs.TestSourceMain'
+}

airbyte-integrations/bases/base-standard-source-test-file/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+/app/bin/"${APPLICATION}" "$@"

airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java

@@ -22,16 +22,18 @@
  * SOFTWARE.
  */
 
-package io.airbyte.integrations.standardtest.source;
+package io.airbyte.integrations.standardtest.source.fs;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.integrations.standardtest.source.StandardSourceTest;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.ConnectorSpecification;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 
 /**
  * Extends TestSource such that it can be called using resources pulled from the file system. Will
@@ -45,6 +47,7 @@ public class ExecutableTestSource extends StandardSourceTest {
     private final Path specPath;
     private final Path configPath;
     private final Path catalogPath;
+
     private final Path statePath;
 
     public TestConfig(String imageName, Path specPath, Path configPath, Path catalogPath, Path statePath) {
@@ -71,6 +74,7 @@ public class ExecutableTestSource extends StandardSourceTest {
       return catalogPath;
     }
 
+    @Nullable
     public Path getStatePath() {
       return statePath;
     }
@@ -101,7 +105,12 @@ public class ExecutableTestSource extends StandardSourceTest {
 
   @Override
   protected JsonNode getState() {
-    return Jsons.deserialize(IOs.readFile(TEST_CONFIG.getStatePath()));
+    if (TEST_CONFIG.getStatePath() != null) {
+      return Jsons.deserialize(IOs.readFile(TEST_CONFIG.getStatePath()));
+    } else {
+      return Jsons.deserialize("{}");
+    }
+
   }
 
   @Override

airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/TestSourceMain.java

@@ -22,9 +22,9 @@
  * SOFTWARE.
  */
 
-package io.airbyte.integrations.standardtest.source;
+package io.airbyte.integrations.standardtest.source.fs;
 
-import io.airbyte.integrations.standardtest.source.ExecutableTestSource.TestConfig;
+import io.airbyte.integrations.standardtest.source.TestRunner;
 import java.nio.file.Path;
 import net.sourceforge.argparse4j.ArgumentParsers;
 import net.sourceforge.argparse4j.inf.ArgumentParser;
@@ -47,18 +47,23 @@ public class TestSourceMain {
         .description("Run standard source tests");
 
     parser.addArgument("--imageName")
-        .help("Name of the integration image");
+        .required(true)
+        .help("Name of the source connector image e.g: airbyte/source-mailchimp");
 
     parser.addArgument("--spec")
+        .required(true)
         .help("Path to file that contains spec json");
 
     parser.addArgument("--config")
+        .required(true)
         .help("Path to file that contains config json");
 
     parser.addArgument("--catalog")
+        .required(true)
         .help("Path to file that contains catalog json");
 
     parser.addArgument("--state")
+        .required(false)
         .help("Path to the file containing state");
 
     Namespace ns = null;
@@ -74,7 +79,13 @@ public class TestSourceMain {
     final String configFile = ns.getString("config");
     final String catalogFile = ns.getString("catalog");
     final String stateFile = ns.getString("state");
-    ExecutableTestSource.TEST_CONFIG = new TestConfig(imageName, Path.of(specFile), Path.of(configFile), Path.of(catalogFile), Path.of(stateFile));
+
+    ExecutableTestSource.TEST_CONFIG = new ExecutableTestSource.TestConfig(
+        imageName,
+        Path.of(specFile),
+        Path.of(configFile),
+        Path.of(catalogFile),
+        stateFile != null ? Path.of(stateFile) : null);
 
     TestRunner.runTestClass(ExecutableTestSource.class);
   }

buildSrc/src/main/groovy/airbyte-standard-source-test-file.gradle

@@ -0,0 +1,73 @@
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+
+abstract class AirbyteStandardSourceTestFileConfiguration {
+    String configPath
+    String configuredCatalogPath
+    String specPath
+    String statePath
+}
+
+class AirbyteStandardSourceTestFilePlugin implements Plugin<Project> {
+    static void assertNotNull(String param, String paramName) {
+        if (param == null || param == "") {
+            throw new IllegalArgumentException("${paramName} parameter must be provided")
+        }
+    }
+
+    static void validateConfig(AirbyteStandardSourceTestFileConfiguration config) {
+        assertNotNull(config.configPath, 'configPath')
+        assertNotNull(config.specPath, 'specPath')
+        assertNotNull(config.configuredCatalogPath, 'configuredCatalogPath')
+    }
+
+    void apply(Project project) {
+        def config = project.extensions.create('airbyteStandardFileSourceTest', AirbyteStandardSourceTestFileConfiguration)
+
+        project.task('standardFsSourceTest') {
+            doFirst {
+                project.exec {
+                    validateConfig(config)
+                    def targetMountDirectory = "/test_input"
+                    def args = [
+                            'docker', 'run', '--rm', '-i',
+                            // provide access to the docker daemon
+                            '-v', "/var/run/docker.sock:/var/run/docker.sock",
+                            // A container within a container mounts from the host filesystem, not the parent container.
+                            // this forces /tmp to be the same directory for host, parent container, and child container.
+                            '-v', "/tmp:/tmp",
+                            // mount the project dir. all provided input paths must be relative to that dir.
+                            '-v', "${project.projectDir.absolutePath}:${targetMountDirectory}",
+                            '--name', "std-fs-source-test-${project.name}",
+                            'airbyte/base-standard-source-test-file:dev',
+                            '--imageName', DockerHelpers.getDevTaggedImage(project.projectDir, 'Dockerfile'),
+                            '--catalog', "${targetMountDirectory}/${config.configuredCatalogPath}",
+                            '--spec', "${targetMountDirectory}/${config.specPath}",
+                            '--config', "${targetMountDirectory}/${config.configPath}",
+                    ]
+
+                    if (config.statePath != null){
+                        args.add("--state")
+                        args.add("${targetMountDirectory}/${config.statePath}")
+                    }
+
+                    commandLine args
+                }
+            }
+
+            outputs.upToDateWhen { false }
+        }
+
+        project.standardFsSourceTest.dependsOn(':airbyte-integrations:bases:base-standard-fs-source-test:airbyteDocker')
+        project.standardFsSourceTest.dependsOn(project.build)
+        project.standardFsSourceTest.dependsOn(project.airbyteDocker)
+
+        // make sure we create the integrationTest task once in case a java integration test was already initialized
+        if (!project.hasProperty('integrationTest')) {
+            project.task('integrationTest')
+        }
+
+        project.integrationTest.dependsOn(project.standardFsSourceTest)
+    }
+}
+

settings.gradle

@@ -30,6 +30,7 @@ include ':airbyte-integrations:bases:base-normalization'
 include ':airbyte-integrations:bases:base-python'
 include ':airbyte-integrations:bases:base-python-test'
 include ':airbyte-integrations:bases:base-singer'
+include ':airbyte-integrations:bases:base-standard-source-test-file'
 include ':airbyte-integrations:bases:standard-destination-test'
 include ':airbyte-integrations:bases:standard-source-test'
 include ':airbyte-integrations:connector-templates:generator'
@@ -54,3 +55,4 @@ integrationsPath.eachDir { dir ->
         include ":airbyte-integrations:connectors:${dir.getFileName()}"
     }
 }
+