Changes project codestyle (#139)
This commit is contained in:
1162
.editorconfig
Normal file
1162
.editorconfig
Normal file
File diff suppressed because it is too large
Load Diff
16
build.gradle
16
build.gradle
@@ -1,6 +1,6 @@
|
||||
plugins {
|
||||
id "base"
|
||||
id "com.diffplug.spotless" version "5.1.1" apply false
|
||||
id "com.diffplug.spotless" version "5.3.0" apply false
|
||||
}
|
||||
|
||||
Properties env = new Properties()
|
||||
@@ -47,16 +47,24 @@ subprojects {
|
||||
|
||||
spotless {
|
||||
java {
|
||||
importOrder()
|
||||
|
||||
eclipse('4.16.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml'))
|
||||
|
||||
licenseHeaderFile createLicenseWith(rootProject.file('LICENSE'))
|
||||
googleJavaFormat('1.8')
|
||||
removeUnusedImports()
|
||||
trimTrailingWhitespace()
|
||||
}
|
||||
groovyGradle {
|
||||
target '**/*.gradle'
|
||||
}
|
||||
format 'json', {
|
||||
target "**/*.json"
|
||||
sql {
|
||||
target '**/*.sql'
|
||||
|
||||
dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties'))
|
||||
}
|
||||
format 'styling', {
|
||||
target '**/*.json', '**/*.yaml'
|
||||
|
||||
prettier()
|
||||
}
|
||||
|
||||
@@ -35,8 +35,8 @@ public class Enums {
|
||||
return Enum.valueOf(oe, ie.name());
|
||||
}
|
||||
|
||||
public static <T1 extends Enum<T1>, T2 extends Enum<T2>> boolean isCompatible(
|
||||
Class<T1> c1, Class<T2> c2) {
|
||||
public static <T1 extends Enum<T1>, T2 extends Enum<T2>> boolean isCompatible(Class<T1> c1,
|
||||
Class<T2> c2) {
|
||||
Preconditions.checkArgument(c1.isEnum());
|
||||
Preconditions.checkArgument(c2.isEnum());
|
||||
return c1.getEnumConstants().length == c2.getEnumConstants().length
|
||||
@@ -45,4 +45,5 @@ public class Enums {
|
||||
Arrays.stream(c2.getEnumConstants()).map(Enum::name).collect(Collectors.toSet()))
|
||||
.isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ package io.dataline.commons.env;
|
||||
import java.util.Objects;
|
||||
|
||||
public enum Env {
|
||||
|
||||
TEST,
|
||||
DOCKER;
|
||||
|
||||
@@ -40,4 +41,5 @@ public enum Env {
|
||||
public static boolean isDocker() {
|
||||
return CURRENT_ENV == Env.DOCKER;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -82,4 +82,5 @@ public class Jsons {
|
||||
public static <T> JsonNode jsonNode(final T object) {
|
||||
return OBJECT_MAPPER.valueToTree(object);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,8 +29,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class HelloWorldTest {
|
||||
|
||||
@Test
|
||||
void name() {
|
||||
assertEquals(1, 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -85,4 +85,5 @@ class EnumsTest {
|
||||
void testNotCompatibleDifferentLength2() {
|
||||
Assertions.assertFalse(isCompatible(E4.class, E1.class));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -145,5 +145,7 @@ class JsonsTest {
|
||||
public int hashCode() {
|
||||
return Objects.hash(str, num, numLong);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,5 +25,7 @@
|
||||
package io.dataline.config.init;
|
||||
|
||||
public class DatalineConfigInitConstants {
|
||||
|
||||
public static final String PLACEHOLDER = "";
|
||||
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
package io.dataline.config;
|
||||
|
||||
public enum ConfigSchema {
|
||||
|
||||
// workspace
|
||||
STANDARD_WORKSPACE("StandardWorkspace.json"),
|
||||
|
||||
@@ -63,4 +64,5 @@ public enum ConfigSchema {
|
||||
public static String getSchemaDirectory() {
|
||||
return "json/";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -35,4 +35,5 @@ public interface Configs {
|
||||
String getWorkspaceDockerMount();
|
||||
|
||||
String getDockerNetwork();
|
||||
|
||||
}
|
||||
|
||||
@@ -90,4 +90,5 @@ public class EnvConfigs implements Configs {
|
||||
}
|
||||
return Path.of(value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -90,4 +90,5 @@ class EnvConfigsTest {
|
||||
when(function.apply(EnvConfigs.DOCKER_NETWORK)).thenReturn("abc");
|
||||
Assertions.assertEquals("abc", config.getDockerNetwork());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
package io.dataline.config.persistence;
|
||||
|
||||
public class ConfigNotFoundException extends Exception {
|
||||
|
||||
public ConfigNotFoundException(String message) {
|
||||
super(message);
|
||||
}
|
||||
@@ -32,4 +33,5 @@ public class ConfigNotFoundException extends Exception {
|
||||
public ConfigNotFoundException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ package io.dataline.config.persistence;
|
||||
import java.util.Set;
|
||||
|
||||
public interface ConfigPersistence {
|
||||
|
||||
<T> T getConfig(PersistenceConfigType persistenceConfigType, String configId, Class<T> clazz)
|
||||
throws ConfigNotFoundException, JsonValidationException;
|
||||
|
||||
@@ -35,4 +36,5 @@ public interface ConfigPersistence {
|
||||
|
||||
<T> void writeConfig(PersistenceConfigType persistenceConfigType, String configId, T config)
|
||||
throws JsonValidationException;
|
||||
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ import org.apache.commons.io.FileUtils;
|
||||
|
||||
// we force all interaction with disk storage to be effectively single threaded.
|
||||
public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
|
||||
private static final String CONFIG_PATH_IN_JAR = "/json";
|
||||
private static final String CONFIG_DIR = "schemas";
|
||||
private static final Path configFilesRoot = getConfigFiles();
|
||||
@@ -98,16 +99,18 @@ public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T getConfig(
|
||||
PersistenceConfigType persistenceConfigType, String configId, Class<T> clazz)
|
||||
public <T> T getConfig(PersistenceConfigType persistenceConfigType,
|
||||
String configId,
|
||||
Class<T> clazz)
|
||||
throws ConfigNotFoundException, JsonValidationException {
|
||||
synchronized (lock) {
|
||||
return getConfigInternal(persistenceConfigType, configId, clazz);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T getConfigInternal(
|
||||
PersistenceConfigType persistenceConfigType, String configId, Class<T> clazz)
|
||||
private <T> T getConfigInternal(PersistenceConfigType persistenceConfigType,
|
||||
String configId,
|
||||
Class<T> clazz)
|
||||
throws ConfigNotFoundException, JsonValidationException {
|
||||
// validate file with schema
|
||||
try {
|
||||
@@ -139,8 +142,9 @@ public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> void writeConfig(
|
||||
PersistenceConfigType persistenceConfigType, String configId, T config)
|
||||
public <T> void writeConfig(PersistenceConfigType persistenceConfigType,
|
||||
String configId,
|
||||
T config)
|
||||
throws JsonValidationException {
|
||||
synchronized (lock) {
|
||||
// validate config with schema
|
||||
@@ -211,8 +215,7 @@ public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
return String.format("%s.json", id);
|
||||
}
|
||||
|
||||
private ConfigSchema standardConfigTypeToConfigSchema(
|
||||
PersistenceConfigType persistenceConfigType) {
|
||||
private ConfigSchema standardConfigTypeToConfigSchema(PersistenceConfigType persistenceConfigType) {
|
||||
switch (persistenceConfigType) {
|
||||
case STANDARD_WORKSPACE:
|
||||
return ConfigSchema.STANDARD_WORKSPACE;
|
||||
@@ -259,8 +262,7 @@ public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
throws ConfigNotFoundException {
|
||||
return getFile(persistenceConfigType, configId)
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new ConfigNotFoundException(
|
||||
() -> new ConfigNotFoundException(
|
||||
String.format(
|
||||
"config type: %s id: %s not found in path %s",
|
||||
persistenceConfigType,
|
||||
@@ -275,4 +277,5 @@ public class DefaultConfigPersistence implements ConfigPersistence {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class JsonSchemaValidation {
|
||||
|
||||
private final SchemaValidatorsConfig schemaValidatorsConfig;
|
||||
private final JsonSchemaFactory jsonSchemaFactory;
|
||||
|
||||
@@ -62,4 +63,5 @@ public class JsonSchemaValidation {
|
||||
configJson.toPrettyString()));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
package io.dataline.config.persistence;
|
||||
|
||||
public class JsonValidationException extends Exception {
|
||||
|
||||
public JsonValidationException(String message) {
|
||||
super(message);
|
||||
}
|
||||
@@ -32,4 +33,5 @@ public class JsonValidationException extends Exception {
|
||||
public JsonValidationException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,8 +27,10 @@ package io.dataline.config.persistence;
|
||||
import java.util.UUID;
|
||||
|
||||
public class PersistenceConstants {
|
||||
|
||||
public static String DEFAULT_TEST_ROOT = "/tmp/data/config/";
|
||||
|
||||
// for MVP we only support one workspace per deployment and we hard code its id.
|
||||
public static UUID DEFAULT_WORKSPACE_ID = UUID.fromString("5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6");
|
||||
|
||||
}
|
||||
|
||||
@@ -191,4 +191,5 @@ class DefaultConfigPersistenceTest {
|
||||
}
|
||||
fail("expected to throw invalid json exception.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,5 +29,7 @@ import org.jooq.DSLContext;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ContextExecutionFunction {
|
||||
|
||||
void execute(DSLContext context) throws SQLException;
|
||||
|
||||
}
|
||||
|
||||
@@ -29,5 +29,7 @@ import org.jooq.DSLContext;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ContextQueryFunction<T> {
|
||||
|
||||
T apply(DSLContext context) throws SQLException;
|
||||
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DatabaseHelper {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseHelper.class);
|
||||
|
||||
public static BasicDataSource getConnectionPoolFromEnv() {
|
||||
@@ -43,8 +44,9 @@ public class DatabaseHelper {
|
||||
System.getenv("DATABASE_URL"));
|
||||
}
|
||||
|
||||
public static BasicDataSource getConnectionPool(
|
||||
String username, String password, String jdbcConnectionString) {
|
||||
public static BasicDataSource getConnectionPool(String username,
|
||||
String password,
|
||||
String jdbcConnectionString) {
|
||||
|
||||
BasicDataSource connectionPool = new BasicDataSource();
|
||||
connectionPool.setDriverClassName("org.postgresql.Driver");
|
||||
@@ -67,8 +69,8 @@ public class DatabaseHelper {
|
||||
}
|
||||
}
|
||||
|
||||
public static void execute(
|
||||
BasicDataSource connectionPool, ContextExecutionFunction executionFunction)
|
||||
public static void execute(BasicDataSource connectionPool,
|
||||
ContextExecutionFunction executionFunction)
|
||||
throws SQLException {
|
||||
query(
|
||||
connectionPool,
|
||||
@@ -77,4 +79,5 @@ public class DatabaseHelper {
|
||||
return 1;
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -33,9 +33,10 @@ import org.jooq.Record;
|
||||
import org.jooq.Result;
|
||||
|
||||
/*
|
||||
The server UUID identifies a specific database installation of Dataline for analytics purposes.
|
||||
* The server UUID identifies a specific database installation of Dataline for analytics purposes.
|
||||
*/
|
||||
public class ServerUuid {
|
||||
|
||||
public static Optional<String> get(BasicDataSource connectionPool) throws SQLException {
|
||||
return DatabaseHelper.query(
|
||||
connectionPool,
|
||||
@@ -51,4 +52,5 @@ public class ServerUuid {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,31 +1,55 @@
|
||||
-- database
|
||||
CREATE DATABASE dataline;
|
||||
CREATE
|
||||
DATABASE dataline;
|
||||
|
||||
\connect dataline;
|
||||
|
||||
-- extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE
|
||||
EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- types
|
||||
CREATE TYPE JOB_STATUS AS ENUM ('pending', 'running', 'failed', 'completed', 'cancelled');
|
||||
CREATE
|
||||
TYPE JOB_STATUS AS ENUM(
|
||||
'pending',
|
||||
'running',
|
||||
'failed',
|
||||
'completed',
|
||||
'cancelled'
|
||||
);
|
||||
|
||||
-- tables
|
||||
CREATE TABLE DATALINE_METADATA (key VARCHAR (255) PRIMARY KEY, value VARCHAR (255));
|
||||
CREATE
|
||||
TABLE
|
||||
DATALINE_METADATA(
|
||||
KEY VARCHAR(255) PRIMARY KEY,
|
||||
value VARCHAR(255)
|
||||
);
|
||||
|
||||
CREATE TABLE JOBS (
|
||||
CREATE
|
||||
TABLE
|
||||
JOBS(
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
scope VARCHAR (255),
|
||||
SCOPE VARCHAR(255),
|
||||
created_at TIMESTAMPTZ,
|
||||
started_at TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ,
|
||||
status JOB_STATUS,
|
||||
config JSONB,
|
||||
output JSONB,
|
||||
OUTPUT JSONB,
|
||||
stdout_path VARCHAR(255),
|
||||
stderr_path VARCHAR(255)
|
||||
);
|
||||
|
||||
-- entries
|
||||
INSERT INTO DATALINE_METADATA VALUES ('server-uuid', uuid_generate_v4());
|
||||
INSERT
|
||||
INTO
|
||||
DATALINE_METADATA
|
||||
VALUES(
|
||||
'server-uuid',
|
||||
uuid_generate_v4()
|
||||
);
|
||||
|
||||
-- grants
|
||||
GRANT ALL ON DATABASE dataline TO docker;
|
||||
GRANT ALL ON
|
||||
DATABASE dataline TO docker;
|
||||
|
||||
@@ -37,6 +37,7 @@ import org.testcontainers.containers.PostgreSQLContainer;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
public class TestServerUuid {
|
||||
|
||||
private static PostgreSQLContainer container;
|
||||
private static BasicDataSource connectionPool;
|
||||
|
||||
@@ -46,8 +47,7 @@ public class TestServerUuid {
|
||||
new PostgreSQLContainer("postgres:13-alpine")
|
||||
.withDatabaseName("dataline")
|
||||
.withUsername("docker")
|
||||
.withPassword("docker");
|
||||
;
|
||||
.withPassword("docker");;
|
||||
container.start();
|
||||
|
||||
try {
|
||||
@@ -82,4 +82,5 @@ public class TestServerUuid {
|
||||
|
||||
assertEquals(uuid1, uuid2);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ package io.dataline.integrations;
|
||||
import java.util.UUID;
|
||||
|
||||
public enum Integrations {
|
||||
|
||||
POSTGRES_TAP(
|
||||
UUID.fromString("2168516a-5c9a-4582-90dc-5e3a01e3f607"),
|
||||
new IntegrationMapping("dataline/integration-singer-postgres-source")),
|
||||
@@ -70,6 +71,7 @@ public enum Integrations {
|
||||
}
|
||||
|
||||
public static class IntegrationMapping {
|
||||
|
||||
private final String checkConnection;
|
||||
private final String discoverSchema;
|
||||
private final String sync;
|
||||
@@ -95,5 +97,7 @@ public enum Integrations {
|
||||
public String getSync() {
|
||||
return sync;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ import java.util.UUID;
|
||||
*/
|
||||
public class ConfigFetchers {
|
||||
|
||||
public static SourceConnectionImplementation getSourceConnectionImplementation(
|
||||
ConfigPersistence configPersistence, UUID sourceImplementationId) {
|
||||
public static SourceConnectionImplementation getSourceConnectionImplementation(ConfigPersistence configPersistence,
|
||||
UUID sourceImplementationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.SOURCE_CONNECTION_IMPLEMENTATION,
|
||||
@@ -58,8 +58,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static DestinationConnectionImplementation getDestinationConnectionImplementation(
|
||||
ConfigPersistence configPersistence, UUID destinationImplementationId) {
|
||||
public static DestinationConnectionImplementation getDestinationConnectionImplementation(ConfigPersistence configPersistence,
|
||||
UUID destinationImplementationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.DESTINATION_CONNECTION_IMPLEMENTATION,
|
||||
@@ -73,8 +73,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardSync getStandardSync(
|
||||
ConfigPersistence configPersistence, UUID connectionId) {
|
||||
public static StandardSync getStandardSync(ConfigPersistence configPersistence,
|
||||
UUID connectionId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_SYNC, connectionId.toString(), StandardSync.class);
|
||||
@@ -93,8 +93,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardSyncSchedule getStandardSyncSchedule(
|
||||
ConfigPersistence configPersistence, UUID connectionId) {
|
||||
public static StandardSyncSchedule getStandardSyncSchedule(ConfigPersistence configPersistence,
|
||||
UUID connectionId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_SYNC_SCHEDULE,
|
||||
@@ -107,8 +107,9 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
private static RuntimeException getConfigNotFoundException(
|
||||
Throwable e, String configName, UUID id) {
|
||||
private static RuntimeException getConfigNotFoundException(Throwable e,
|
||||
String configName,
|
||||
UUID id) {
|
||||
return new RuntimeException(
|
||||
String.format("Could not find sync configuration for %s: %s.", configName, id), e);
|
||||
}
|
||||
@@ -120,4 +121,5 @@ public class ConfigFetchers {
|
||||
e.getMessage()),
|
||||
e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -82,8 +82,8 @@ public class DefaultSchedulerPersistence implements SchedulerPersistence {
|
||||
}
|
||||
|
||||
@Override
|
||||
public long createDestinationCheckConnectionJob(
|
||||
DestinationConnectionImplementation destinationImplementation) throws IOException {
|
||||
public long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation)
|
||||
throws IOException {
|
||||
final String scope =
|
||||
ScopeHelper.createScope(
|
||||
JobConfig.ConfigType.CHECK_CONNECTION_DESTINATION,
|
||||
@@ -127,8 +127,7 @@ public class DefaultSchedulerPersistence implements SchedulerPersistence {
|
||||
}
|
||||
|
||||
@Override
|
||||
public long createSyncJob(
|
||||
SourceConnectionImplementation sourceImplementation,
|
||||
public long createSyncJob(SourceConnectionImplementation sourceImplementation,
|
||||
DestinationConnectionImplementation destinationImplementation,
|
||||
StandardSync standardSync)
|
||||
throws IOException {
|
||||
@@ -172,8 +171,7 @@ public class DefaultSchedulerPersistence implements SchedulerPersistence {
|
||||
record =
|
||||
DatabaseHelper.query(
|
||||
connectionPool,
|
||||
ctx ->
|
||||
ctx.fetch(
|
||||
ctx -> ctx.fetch(
|
||||
"INSERT INTO jobs(scope, created_at, updated_at, status, config, output, stdout_path, stderr_path) VALUES(?, ?, ?, CAST(? AS JOB_STATUS), CAST(? as JSONB), ?, ?, ?) RETURNING id",
|
||||
scope,
|
||||
now,
|
||||
@@ -218,8 +216,7 @@ public class DefaultSchedulerPersistence implements SchedulerPersistence {
|
||||
String scope = ScopeHelper.createScope(configType, configId);
|
||||
return DatabaseHelper.query(
|
||||
connectionPool,
|
||||
ctx ->
|
||||
ctx.fetch("SELECT * FROM jobs WHERE scope = ?", scope).stream()
|
||||
ctx -> ctx.fetch("SELECT * FROM jobs WHERE scope = ?", scope).stream()
|
||||
.map(DefaultSchedulerPersistence::getJobFromRecord)
|
||||
.collect(Collectors.toList()));
|
||||
} catch (SQLException e) {
|
||||
@@ -252,4 +249,5 @@ public class DefaultSchedulerPersistence implements SchedulerPersistence {
|
||||
private static long getEpoch(Record record, String fieldName) {
|
||||
return record.getValue(fieldName, LocalDateTime.class).toEpochSecond(ZoneOffset.UTC);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -42,8 +42,7 @@ public class Job {
|
||||
private final String stdoutPath;
|
||||
private final String stderrPath;
|
||||
|
||||
public Job(
|
||||
long id,
|
||||
public Job(long id,
|
||||
String scope,
|
||||
JobStatus status,
|
||||
JobConfig config,
|
||||
@@ -134,4 +133,5 @@ public class Job {
|
||||
+ '\''
|
||||
+ '}';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,10 +27,12 @@ package io.dataline.scheduler;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
public class JobLogs {
|
||||
|
||||
public static String ROOT_PATH = "logs/jobs";
|
||||
|
||||
// todo: add scoping by job id
|
||||
public static String getLogDirectory(String scope) {
|
||||
return Paths.get(ROOT_PATH, scope).toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -46,8 +46,7 @@ public class JobScheduler implements Runnable {
|
||||
private final SchedulerPersistence schedulerPersistence;
|
||||
private final ConfigPersistence configPersistence;
|
||||
|
||||
public JobScheduler(
|
||||
BasicDataSource connectionPool,
|
||||
public JobScheduler(BasicDataSource connectionPool,
|
||||
SchedulerPersistence schedulerPersistence,
|
||||
ConfigPersistence configPersistence) {
|
||||
this.connectionPool = connectionPool;
|
||||
@@ -138,4 +137,5 @@ public class JobScheduler implements Runnable {
|
||||
private Set<StandardSync> getAllActiveConnections() {
|
||||
return ConfigFetchers.getStandardSyncs(configPersistence);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import com.google.common.collect.Sets;
|
||||
import java.util.Set;
|
||||
|
||||
public enum JobStatus {
|
||||
|
||||
PENDING,
|
||||
RUNNING,
|
||||
FAILED,
|
||||
@@ -35,4 +36,5 @@ public enum JobStatus {
|
||||
CANCELLED;
|
||||
|
||||
public static Set<JobStatus> TERMINAL_STATUSES = Sets.newHashSet(FAILED, COMPLETED, CANCELLED);
|
||||
|
||||
}
|
||||
|
||||
@@ -45,8 +45,7 @@ public class JobSubmitter implements Runnable {
|
||||
private final Path workspaceRoot;
|
||||
private final ProcessBuilderFactory pbf;
|
||||
|
||||
public JobSubmitter(
|
||||
final ExecutorService threadPool,
|
||||
public JobSubmitter(final ExecutorService threadPool,
|
||||
final BasicDataSource connectionPool,
|
||||
final SchedulerPersistence persistence,
|
||||
final Path workspaceRoot,
|
||||
@@ -97,4 +96,5 @@ public class JobSubmitter implements Runnable {
|
||||
threadPool.submit(
|
||||
new WorkerRunner(job.getId(), connectionPool, persistence, workspaceRoot, pbf));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ import org.apache.commons.dbcp2.BasicDataSource;
|
||||
import org.jooq.Record;
|
||||
|
||||
public class JobUtils {
|
||||
public static long createSyncJobFromConnectionId(
|
||||
SchedulerPersistence schedulerPersistence,
|
||||
|
||||
public static long createSyncJobFromConnectionId(SchedulerPersistence schedulerPersistence,
|
||||
ConfigPersistence configPersistence,
|
||||
UUID connectionId) {
|
||||
final StandardSync standardSync;
|
||||
@@ -59,8 +59,9 @@ public class JobUtils {
|
||||
}
|
||||
}
|
||||
|
||||
public static Optional<Job> getLastSyncJobForConnectionId(
|
||||
BasicDataSource connectionPool, UUID connectionId) throws IOException {
|
||||
public static Optional<Job> getLastSyncJobForConnectionId(BasicDataSource connectionPool,
|
||||
UUID connectionId)
|
||||
throws IOException {
|
||||
try {
|
||||
return DatabaseHelper.query(
|
||||
connectionPool,
|
||||
@@ -86,4 +87,5 @@ public class JobUtils {
|
||||
throw new IOException(throwables);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -62,8 +62,7 @@ public class SchedulerApp {
|
||||
private final Path workspaceRoot;
|
||||
private final ProcessBuilderFactory pbf;
|
||||
|
||||
public SchedulerApp(
|
||||
BasicDataSource connectionPool,
|
||||
public SchedulerApp(BasicDataSource connectionPool,
|
||||
Path configRoot,
|
||||
Path workspaceRoot,
|
||||
ProcessBuilderFactory pbf) {
|
||||
@@ -119,4 +118,5 @@ public class SchedulerApp {
|
||||
LOGGER.info("Launching scheduler...");
|
||||
new SchedulerApp(connectionPool, configRoot, workspaceRoot, pbf).start();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -32,17 +32,17 @@ import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
public interface SchedulerPersistence {
|
||||
|
||||
long createSourceCheckConnectionJob(SourceConnectionImplementation sourceImplementation)
|
||||
throws IOException;
|
||||
|
||||
long createDestinationCheckConnectionJob(
|
||||
DestinationConnectionImplementation destinationImplementation) throws IOException;
|
||||
long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation)
|
||||
throws IOException;
|
||||
|
||||
long createDiscoverSchemaJob(SourceConnectionImplementation sourceImplementation)
|
||||
throws IOException;
|
||||
|
||||
long createSyncJob(
|
||||
SourceConnectionImplementation sourceImplementation,
|
||||
long createSyncJob(SourceConnectionImplementation sourceImplementation,
|
||||
DestinationConnectionImplementation destinationImplementation,
|
||||
StandardSync standardSync)
|
||||
throws IOException;
|
||||
@@ -50,4 +50,5 @@ public interface SchedulerPersistence {
|
||||
Job getJob(long jobId) throws IOException;
|
||||
|
||||
List<Job> listJobs(JobConfig.ConfigType configType, String configId) throws IOException;
|
||||
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class SchedulerShutdownHandler extends Thread {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SchedulerShutdownHandler.class);
|
||||
private ExecutorService[] threadPools;
|
||||
|
||||
@@ -51,4 +52,5 @@ public class SchedulerShutdownHandler extends Thread {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ package io.dataline.scheduler;
|
||||
import io.dataline.config.JobConfig;
|
||||
|
||||
public class ScopeHelper {
|
||||
|
||||
public static String createScope(JobConfig.ConfigType configType, String configId) {
|
||||
return configType.value() + ":" + configId;
|
||||
}
|
||||
@@ -34,4 +35,5 @@ public class ScopeHelper {
|
||||
public static String getConfigId(String scope) {
|
||||
return scope.split(":")[1];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -43,8 +43,9 @@ import org.slf4j.LoggerFactory;
|
||||
* outputs are passed to the selected worker. It also makes sures that the outputs of the worker are
|
||||
* persisted to the db.
|
||||
*
|
||||
* <p>todo (cgardens) - this line between this abstraction and WorkerRunner is a little blurry. we
|
||||
* can clarify it later. the main benefit is of this class is that it gives us some type safety when
|
||||
* <p>
|
||||
* todo (cgardens) - this line between this abstraction and WorkerRunner is a little blurry. we can
|
||||
* clarify it later. the main benefit is of this class is that it gives us some type safety when
|
||||
* working with workers. you can probably make an argument that this class should not have access to
|
||||
* the db.
|
||||
*
|
||||
@@ -61,8 +62,7 @@ public class WorkerRun<InputType, OutputType> implements Runnable {
|
||||
private final Worker<InputType, OutputType> worker;
|
||||
private final BasicDataSource connectionPool;
|
||||
|
||||
public WorkerRun(
|
||||
long jobId,
|
||||
public WorkerRun(long jobId,
|
||||
Path jobRoot,
|
||||
InputType input,
|
||||
Worker<InputType, OutputType> worker,
|
||||
@@ -117,8 +117,7 @@ public class WorkerRun<InputType, OutputType> implements Runnable {
|
||||
try {
|
||||
DatabaseHelper.query(
|
||||
connectionPool,
|
||||
ctx ->
|
||||
ctx.execute(
|
||||
ctx -> ctx.execute(
|
||||
"UPDATE jobs SET status = CAST(? as JOB_STATUS), updated_at = ? WHERE id = ?",
|
||||
status.toString().toLowerCase(),
|
||||
now,
|
||||
@@ -135,8 +134,7 @@ public class WorkerRun<InputType, OutputType> implements Runnable {
|
||||
try {
|
||||
DatabaseHelper.query(
|
||||
connectionPool,
|
||||
ctx ->
|
||||
ctx.execute(
|
||||
ctx -> ctx.execute(
|
||||
"UPDATE jobs SET output = CAST(? as JSONB), updated_at = ? WHERE id = ?",
|
||||
outputJson,
|
||||
now,
|
||||
@@ -146,4 +144,5 @@ public class WorkerRun<InputType, OutputType> implements Runnable {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -52,8 +52,7 @@ public class WorkerRunner implements Runnable {
|
||||
private final Path workspaceRoot;
|
||||
private final ProcessBuilderFactory pbf;
|
||||
|
||||
public WorkerRunner(
|
||||
long jobId,
|
||||
public WorkerRunner(long jobId,
|
||||
BasicDataSource connectionPool,
|
||||
SchedulerPersistence persistence,
|
||||
Path workspaceRoot,
|
||||
@@ -124,16 +123,14 @@ public class WorkerRunner implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
private static StandardCheckConnectionInput getCheckConnectionInput(
|
||||
JobCheckConnectionConfig config) {
|
||||
private static StandardCheckConnectionInput getCheckConnectionInput(JobCheckConnectionConfig config) {
|
||||
final StandardCheckConnectionInput checkConnectionInput = new StandardCheckConnectionInput();
|
||||
checkConnectionInput.setConnectionConfigurationJson(config.getConnectionConfigurationJson());
|
||||
|
||||
return checkConnectionInput;
|
||||
}
|
||||
|
||||
private static StandardDiscoverSchemaInput getDiscoverSchemaInput(
|
||||
JobDiscoverSchemaConfig config) {
|
||||
private static StandardDiscoverSchemaInput getDiscoverSchemaInput(JobDiscoverSchemaConfig config) {
|
||||
final StandardDiscoverSchemaInput discoverSchemaInput = new StandardDiscoverSchemaInput();
|
||||
discoverSchemaInput.setConnectionConfigurationJson(config.getConnectionConfigurationJson());
|
||||
|
||||
@@ -149,4 +146,5 @@ public class WorkerRunner implements Runnable {
|
||||
|
||||
return syncInput;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -52,4 +52,5 @@ public class ConfigurationApiFactory implements Factory<ConfigurationApi> {
|
||||
public void dispose(ConfigurationApi service) {
|
||||
/* noop */
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -36,10 +36,11 @@ public class CorsFilter implements ContainerResponseFilter {
|
||||
private static final String ALLOW_METHODS = "Access-Control-Allow-Methods";
|
||||
|
||||
@Override
|
||||
public void filter(
|
||||
ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
|
||||
public void filter(ContainerRequestContext requestContext,
|
||||
ContainerResponseContext responseContext) {
|
||||
responseContext.getHeaders().add(ALLOW_ORIGIN, "*");
|
||||
responseContext.getHeaders().add(ALLOW_HEADERS, "Origin, Content-Type, Accept");
|
||||
responseContext.getHeaders().add(ALLOW_METHODS, "GET, POST, PUT, DELETE, OPTIONS, HEAD");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -78,12 +78,14 @@ public class ServerApp {
|
||||
.register(ConfigurationApi.class)
|
||||
.register(
|
||||
new AbstractBinder() {
|
||||
|
||||
@Override
|
||||
public void configure() {
|
||||
bindFactory(ConfigurationApiFactory.class)
|
||||
.to(ConfigurationApi.class)
|
||||
.in(RequestScoped.class);
|
||||
}
|
||||
|
||||
})
|
||||
// exception handling
|
||||
.register(InvalidInputExceptionMapper.class)
|
||||
@@ -122,4 +124,5 @@ public class ServerApp {
|
||||
LOGGER.info("Starting server...");
|
||||
new ServerApp(configRoot).start();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ import org.apache.commons.dbcp2.BasicDataSource;
|
||||
|
||||
@javax.ws.rs.Path("/v1")
|
||||
public class ConfigurationApi implements io.dataline.api.V1Api {
|
||||
|
||||
private final WorkspacesHandler workspacesHandler;
|
||||
private final SourcesHandler sourcesHandler;
|
||||
private final SourceSpecificationsHandler sourceSpecificationsHandler;
|
||||
@@ -142,53 +143,45 @@ public class ConfigurationApi implements io.dataline.api.V1Api {
|
||||
// SOURCE SPECIFICATION
|
||||
|
||||
@Override
|
||||
public SourceSpecificationRead getSourceSpecification(
|
||||
@Valid SourceIdRequestBody sourceIdRequestBody) {
|
||||
public SourceSpecificationRead getSourceSpecification(@Valid SourceIdRequestBody sourceIdRequestBody) {
|
||||
return sourceSpecificationsHandler.getSourceSpecification(sourceIdRequestBody);
|
||||
}
|
||||
|
||||
// SOURCE IMPLEMENTATION
|
||||
|
||||
@Override
|
||||
public SourceImplementationRead createSourceImplementation(
|
||||
@Valid SourceImplementationCreate sourceImplementationCreate) {
|
||||
public SourceImplementationRead createSourceImplementation(@Valid SourceImplementationCreate sourceImplementationCreate) {
|
||||
return sourceImplementationsHandler.createSourceImplementation(sourceImplementationCreate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SourceImplementationRead updateSourceImplementation(
|
||||
@Valid SourceImplementationUpdate sourceImplementationUpdate) {
|
||||
public SourceImplementationRead updateSourceImplementation(@Valid SourceImplementationUpdate sourceImplementationUpdate) {
|
||||
return sourceImplementationsHandler.updateSourceImplementation(sourceImplementationUpdate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SourceImplementationReadList listSourceImplementationsForWorkspace(
|
||||
@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public SourceImplementationReadList listSourceImplementationsForWorkspace(@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
return sourceImplementationsHandler.listSourceImplementationsForWorkspace(
|
||||
workspaceIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SourceImplementationRead getSourceImplementation(
|
||||
@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public SourceImplementationRead getSourceImplementation(@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
return sourceImplementationsHandler.getSourceImplementation(sourceImplementationIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteSourceImplementation(
|
||||
@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public void deleteSourceImplementation(@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
sourceImplementationsHandler.deleteSourceImplementation(sourceImplementationIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CheckConnectionRead checkConnectionToSourceImplementation(
|
||||
@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public CheckConnectionRead checkConnectionToSourceImplementation(@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
return schedulerHandler.checkSourceImplementationConnection(sourceImplementationIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(
|
||||
@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
return schedulerHandler.discoverSchemaForSourceImplementation(
|
||||
sourceImplementationIdRequestBody);
|
||||
}
|
||||
@@ -208,43 +201,37 @@ public class ConfigurationApi implements io.dataline.api.V1Api {
|
||||
// DESTINATION SPECIFICATION
|
||||
|
||||
@Override
|
||||
public DestinationSpecificationRead getDestinationSpecification(
|
||||
@Valid DestinationIdRequestBody destinationIdRequestBody) {
|
||||
public DestinationSpecificationRead getDestinationSpecification(@Valid DestinationIdRequestBody destinationIdRequestBody) {
|
||||
return destinationSpecificationsHandler.getDestinationSpecification(destinationIdRequestBody);
|
||||
}
|
||||
|
||||
// DESTINATION IMPLEMENTATION
|
||||
@Override
|
||||
public DestinationImplementationRead createDestinationImplementation(
|
||||
@Valid DestinationImplementationCreate destinationImplementationCreate) {
|
||||
public DestinationImplementationRead createDestinationImplementation(@Valid DestinationImplementationCreate destinationImplementationCreate) {
|
||||
return destinationImplementationsHandler.createDestinationImplementation(
|
||||
destinationImplementationCreate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DestinationImplementationRead updateDestinationImplementation(
|
||||
@Valid DestinationImplementationUpdate destinationImplementationUpdate) {
|
||||
public DestinationImplementationRead updateDestinationImplementation(@Valid DestinationImplementationUpdate destinationImplementationUpdate) {
|
||||
return destinationImplementationsHandler.updateDestinationImplementation(
|
||||
destinationImplementationUpdate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DestinationImplementationReadList listDestinationImplementationsForWorkspace(
|
||||
@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public DestinationImplementationReadList listDestinationImplementationsForWorkspace(@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
return destinationImplementationsHandler.listDestinationImplementationsForWorkspace(
|
||||
workspaceIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DestinationImplementationRead getDestinationImplementation(
|
||||
@Valid DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
public DestinationImplementationRead getDestinationImplementation(@Valid DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
return destinationImplementationsHandler.getDestinationImplementation(
|
||||
destinationImplementationIdRequestBody);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CheckConnectionRead checkConnectionToDestinationImplementation(
|
||||
@Valid DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
public CheckConnectionRead checkConnectionToDestinationImplementation(@Valid DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
return schedulerHandler.checkDestinationImplementationConnection(
|
||||
destinationImplementationIdRequestBody);
|
||||
}
|
||||
@@ -262,8 +249,7 @@ public class ConfigurationApi implements io.dataline.api.V1Api {
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectionReadList listConnectionsForWorkspace(
|
||||
@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public ConnectionReadList listConnectionsForWorkspace(@Valid WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
return connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody);
|
||||
}
|
||||
|
||||
@@ -288,4 +274,5 @@ public class ConfigurationApi implements io.dataline.api.V1Api {
|
||||
public JobInfoRead getJobInfo(@Valid JobIdRequestBody jobIdRequestBody) {
|
||||
return jobHistoryHandler.getJobInfo(jobIdRequestBody);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SchemaConverter {
|
||||
|
||||
public static Schema toPersistenceSchema(SourceSchema api) {
|
||||
final List<Table> persistenceTables =
|
||||
api.getTables().stream()
|
||||
@@ -108,4 +109,5 @@ public class SchemaConverter {
|
||||
public static io.dataline.api.model.DataType toApiDataType(DataType persistenceDataType) {
|
||||
return Enums.convertTo(persistenceDataType, io.dataline.api.model.DataType.class);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -64,4 +64,5 @@ public class InvalidInputExceptionMapper implements ExceptionMapper<ConstraintVi
|
||||
}
|
||||
return message.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -43,4 +43,5 @@ public class InvalidJsonExceptionMapper implements ExceptionMapper<JsonParseExce
|
||||
.type("application/json")
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -43,4 +43,5 @@ public class InvalidJsonInputExceptionMapper implements ExceptionMapper<JsonMapp
|
||||
.type("application/json")
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
package io.dataline.server.errors;
|
||||
|
||||
public class KnownException extends RuntimeException {
|
||||
|
||||
private final int httpCode;
|
||||
|
||||
public KnownException(int httpCode, String message) {
|
||||
@@ -40,4 +41,5 @@ public class KnownException extends RuntimeException {
|
||||
public int getHttpCode() {
|
||||
return httpCode;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -45,4 +45,5 @@ public class KnownExceptionMapper implements ExceptionMapper<KnownException> {
|
||||
.type("application/json")
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -45,4 +45,5 @@ public class UncaughtExceptionMapper implements ExceptionMapper<Throwable> {
|
||||
.type("application/json")
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -151,8 +151,7 @@ public class ConnectionsHandler {
|
||||
}
|
||||
|
||||
// todo (cgardens) - this is a disaster without a relational db.
|
||||
public ConnectionReadList listConnectionsForWorkspace(
|
||||
WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public ConnectionReadList listConnectionsForWorkspace(WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
|
||||
final List<ConnectionRead> reads =
|
||||
// read all connections.
|
||||
@@ -160,8 +159,7 @@ public class ConnectionsHandler {
|
||||
// filter out connections attached to source implementations NOT associated with the
|
||||
// workspace
|
||||
.filter(
|
||||
standardSync ->
|
||||
ConfigFetchers.getSourceConnectionImplementation(
|
||||
standardSync -> ConfigFetchers.getSourceConnectionImplementation(
|
||||
configPersistence, standardSync.getSourceImplementationId())
|
||||
.getWorkspaceId()
|
||||
.equals(workspaceIdRequestBody.getWorkspaceId()))
|
||||
@@ -202,8 +200,8 @@ public class ConnectionsHandler {
|
||||
return ConfigFetchers.getStandardSyncSchedule(configPersistence, connectionId);
|
||||
}
|
||||
|
||||
private ConnectionRead toConnectionRead(
|
||||
StandardSync standardSync, StandardSyncSchedule standardSyncSchedule) {
|
||||
private ConnectionRead toConnectionRead(StandardSync standardSync,
|
||||
StandardSyncSchedule standardSyncSchedule) {
|
||||
final ConnectionSchedule apiSchedule;
|
||||
|
||||
standardSyncSchedule.setConnectionId(standardSyncSchedule.getConnectionId());
|
||||
@@ -247,4 +245,5 @@ public class ConnectionsHandler {
|
||||
private ConnectionSchedule.TimeUnitEnum toApiTimeUnit(Schedule.TimeUnit apiTimeUnit) {
|
||||
return Enums.convertTo(apiTimeUnit, ConnectionSchedule.TimeUnitEnum.class);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -48,8 +48,7 @@ public class DestinationImplementationsHandler {
|
||||
private final ConfigPersistence configPersistence;
|
||||
private final IntegrationSchemaValidation validator;
|
||||
|
||||
public DestinationImplementationsHandler(
|
||||
ConfigPersistence configPersistence,
|
||||
public DestinationImplementationsHandler(ConfigPersistence configPersistence,
|
||||
IntegrationSchemaValidation integrationSchemaValidation,
|
||||
Supplier<UUID> uuidGenerator) {
|
||||
this.configPersistence = configPersistence;
|
||||
@@ -57,14 +56,12 @@ public class DestinationImplementationsHandler {
|
||||
this.uuidGenerator = uuidGenerator;
|
||||
}
|
||||
|
||||
public DestinationImplementationsHandler(
|
||||
ConfigPersistence configPersistence,
|
||||
public DestinationImplementationsHandler(ConfigPersistence configPersistence,
|
||||
IntegrationSchemaValidation integrationSchemaValidation) {
|
||||
this(configPersistence, integrationSchemaValidation, UUID::randomUUID);
|
||||
}
|
||||
|
||||
public DestinationImplementationRead createDestinationImplementation(
|
||||
DestinationImplementationCreate destinationImplementationCreate) {
|
||||
public DestinationImplementationRead createDestinationImplementation(DestinationImplementationCreate destinationImplementationCreate) {
|
||||
// validate configuration
|
||||
validateDestinationImplementation(
|
||||
destinationImplementationCreate.getDestinationSpecificationId(),
|
||||
@@ -82,8 +79,7 @@ public class DestinationImplementationsHandler {
|
||||
return getDestinationImplementationInternal(destinationImplementationId);
|
||||
}
|
||||
|
||||
public DestinationImplementationRead updateDestinationImplementation(
|
||||
DestinationImplementationUpdate destinationImplementationUpdate) {
|
||||
public DestinationImplementationRead updateDestinationImplementation(DestinationImplementationUpdate destinationImplementationUpdate) {
|
||||
// get existing implementation
|
||||
final DestinationImplementationRead persistedDestinationImplementation =
|
||||
getDestinationImplementationInternal(
|
||||
@@ -106,20 +102,17 @@ public class DestinationImplementationsHandler {
|
||||
destinationImplementationUpdate.getDestinationImplementationId());
|
||||
}
|
||||
|
||||
public DestinationImplementationRead getDestinationImplementation(
|
||||
DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
public DestinationImplementationRead getDestinationImplementation(DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
|
||||
return getDestinationImplementationInternal(
|
||||
destinationImplementationIdRequestBody.getDestinationImplementationId());
|
||||
}
|
||||
|
||||
public DestinationImplementationReadList listDestinationImplementationsForWorkspace(
|
||||
WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public DestinationImplementationReadList listDestinationImplementationsForWorkspace(WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
final List<DestinationImplementationRead> reads =
|
||||
ConfigFetchers.getDestinationConnectionImplementations(configPersistence).stream()
|
||||
.filter(
|
||||
destinationConnectionImplementation ->
|
||||
destinationConnectionImplementation
|
||||
destinationConnectionImplementation -> destinationConnectionImplementation
|
||||
.getWorkspaceId()
|
||||
.equals(workspaceIdRequestBody.getWorkspaceId()))
|
||||
.map(
|
||||
@@ -140,8 +133,7 @@ public class DestinationImplementationsHandler {
|
||||
return destinationImplementationReadList;
|
||||
}
|
||||
|
||||
private DestinationImplementationRead getDestinationImplementationInternal(
|
||||
UUID destinationImplementationId) {
|
||||
private DestinationImplementationRead getDestinationImplementationInternal(UUID destinationImplementationId) {
|
||||
// read configuration from db
|
||||
final DestinationConnectionImplementation retrievedDestinationConnectionImplementation;
|
||||
retrievedDestinationConnectionImplementation =
|
||||
@@ -158,11 +150,9 @@ public class DestinationImplementationsHandler {
|
||||
retrievedDestinationConnectionImplementation, destinationId);
|
||||
}
|
||||
|
||||
private void validateDestinationImplementation(
|
||||
UUID destinationConnectionSpecificationId, String implementationJson) {
|
||||
private void validateDestinationImplementation(UUID destinationConnectionSpecificationId, String implementationJson) {
|
||||
try {
|
||||
validator.validateDestinationConnectionConfiguration(
|
||||
destinationConnectionSpecificationId, implementationJson);
|
||||
validator.validateDestinationConnectionConfiguration(destinationConnectionSpecificationId, implementationJson);
|
||||
} catch (JsonValidationException e) {
|
||||
throw new KnownException(
|
||||
422,
|
||||
@@ -172,8 +162,7 @@ public class DestinationImplementationsHandler {
|
||||
}
|
||||
}
|
||||
|
||||
private void persistDestinationConnectionImplementation(
|
||||
UUID destinationSpecificationId,
|
||||
private void persistDestinationConnectionImplementation(UUID destinationSpecificationId,
|
||||
UUID workspaceId,
|
||||
UUID destinationImplementationId,
|
||||
String configurationJson) {
|
||||
@@ -191,8 +180,8 @@ public class DestinationImplementationsHandler {
|
||||
destinationConnectionImplementation);
|
||||
}
|
||||
|
||||
private DestinationImplementationRead toDestinationImplementationRead(
|
||||
DestinationConnectionImplementation destinationConnectionImplementation, UUID destinationId) {
|
||||
private DestinationImplementationRead toDestinationImplementationRead(DestinationConnectionImplementation destinationConnectionImplementation,
|
||||
UUID destinationId) {
|
||||
final DestinationImplementationRead destinationImplementationRead =
|
||||
new DestinationImplementationRead();
|
||||
destinationImplementationRead.setDestinationId(destinationId);
|
||||
@@ -207,4 +196,5 @@ public class DestinationImplementationsHandler {
|
||||
|
||||
return destinationImplementationRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -39,8 +39,7 @@ public class DestinationSpecificationsHandler {
|
||||
this.configPersistence = configPersistence;
|
||||
}
|
||||
|
||||
public DestinationSpecificationRead getDestinationSpecification(
|
||||
DestinationIdRequestBody destinationIdRequestBody) {
|
||||
public DestinationSpecificationRead getDestinationSpecification(DestinationIdRequestBody destinationIdRequestBody) {
|
||||
final DestinationConnectionSpecification destinationConnection;
|
||||
// todo (cgardens) - this is a shortcoming of rolling our own disk storage. since we are not
|
||||
// querying on a the primary key, we have to list all of the specification objects and then
|
||||
@@ -48,14 +47,12 @@ public class DestinationSpecificationsHandler {
|
||||
destinationConnection =
|
||||
ConfigFetchers.getDestinationConnectionSpecifications(configPersistence).stream()
|
||||
.filter(
|
||||
destinationSpecification ->
|
||||
destinationSpecification
|
||||
destinationSpecification -> destinationSpecification
|
||||
.getDestinationId()
|
||||
.equals(destinationIdRequestBody.getDestinationId()))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new KnownException(
|
||||
() -> new KnownException(
|
||||
404,
|
||||
String.format(
|
||||
"Could not find a destination specification for destination: %s",
|
||||
@@ -64,8 +61,7 @@ public class DestinationSpecificationsHandler {
|
||||
return toDestinationSpecificationRead(destinationConnection);
|
||||
}
|
||||
|
||||
private static DestinationSpecificationRead toDestinationSpecificationRead(
|
||||
DestinationConnectionSpecification destinationConnectionSpecification) {
|
||||
private static DestinationSpecificationRead toDestinationSpecificationRead(DestinationConnectionSpecification destinationConnectionSpecification) {
|
||||
final DestinationSpecificationRead destinationSpecificationRead =
|
||||
new DestinationSpecificationRead();
|
||||
destinationSpecificationRead.setDestinationId(
|
||||
@@ -77,4 +73,5 @@ public class DestinationSpecificationsHandler {
|
||||
|
||||
return destinationSpecificationRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class DestinationsHandler {
|
||||
|
||||
private final ConfigPersistence configPersistence;
|
||||
|
||||
public DestinationsHandler(ConfigPersistence configPersistence) {
|
||||
@@ -67,4 +68,5 @@ public class DestinationsHandler {
|
||||
|
||||
return destinationRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ import java.util.stream.Collectors;
|
||||
import org.apache.commons.io.input.ReversedLinesFileReader;
|
||||
|
||||
public class JobHistoryHandler {
|
||||
|
||||
private static final int LOG_TAIL_SIZE = 10;
|
||||
private final SchedulerPersistence schedulerPersistence;
|
||||
|
||||
@@ -131,4 +132,5 @@ public class JobHistoryHandler {
|
||||
|
||||
return jobRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -49,20 +49,20 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class SchedulerHandler {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SchedulerHandler.class);
|
||||
|
||||
private final ConfigPersistence configPersistence;
|
||||
private final SchedulerPersistence schedulerPersistence;
|
||||
|
||||
public SchedulerHandler(
|
||||
ConfigPersistence configPersistence, SchedulerPersistence schedulerPersistence) {
|
||||
public SchedulerHandler(ConfigPersistence configPersistence,
|
||||
SchedulerPersistence schedulerPersistence) {
|
||||
|
||||
this.configPersistence = configPersistence;
|
||||
this.schedulerPersistence = schedulerPersistence;
|
||||
}
|
||||
|
||||
public CheckConnectionRead checkSourceImplementationConnection(
|
||||
SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public CheckConnectionRead checkSourceImplementationConnection(SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
|
||||
final SourceConnectionImplementation connectionImplementation =
|
||||
ConfigFetchers.getSourceConnectionImplementation(
|
||||
@@ -80,8 +80,7 @@ public class SchedulerHandler {
|
||||
return reportConnectionStatus(job);
|
||||
}
|
||||
|
||||
public CheckConnectionRead checkDestinationImplementationConnection(
|
||||
DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
public CheckConnectionRead checkDestinationImplementationConnection(DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) {
|
||||
|
||||
final DestinationConnectionImplementation connectionImplementation =
|
||||
ConfigFetchers.getDestinationConnectionImplementation(
|
||||
@@ -100,8 +99,7 @@ public class SchedulerHandler {
|
||||
return reportConnectionStatus(job);
|
||||
}
|
||||
|
||||
public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(
|
||||
SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
final SourceConnectionImplementation connectionImplementation =
|
||||
ConfigFetchers.getSourceConnectionImplementation(
|
||||
configPersistence, sourceImplementationIdRequestBody.getSourceImplementationId());
|
||||
@@ -131,7 +129,8 @@ public class SchedulerHandler {
|
||||
}
|
||||
|
||||
public ConnectionSyncRead syncConnection(ConnectionIdRequestBody connectionIdRequestBody) {
|
||||
@NotNull final UUID connectionId = connectionIdRequestBody.getConnectionId();
|
||||
@NotNull
|
||||
final UUID connectionId = connectionIdRequestBody.getConnectionId();
|
||||
final StandardSync standardSync;
|
||||
standardSync = ConfigFetchers.getStandardSync(configPersistence, connectionId);
|
||||
|
||||
@@ -204,4 +203,5 @@ public class SchedulerHandler {
|
||||
|
||||
return checkConnectionRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -48,8 +48,7 @@ public class SourceImplementationsHandler {
|
||||
private final ConfigPersistence configPersistence;
|
||||
private final IntegrationSchemaValidation validator;
|
||||
|
||||
public SourceImplementationsHandler(
|
||||
ConfigPersistence configPersistence,
|
||||
public SourceImplementationsHandler(ConfigPersistence configPersistence,
|
||||
IntegrationSchemaValidation integrationSchemaValidation,
|
||||
Supplier<UUID> uuidGenerator) {
|
||||
this.configPersistence = configPersistence;
|
||||
@@ -57,14 +56,12 @@ public class SourceImplementationsHandler {
|
||||
this.uuidGenerator = uuidGenerator;
|
||||
}
|
||||
|
||||
public SourceImplementationsHandler(
|
||||
ConfigPersistence configPersistence,
|
||||
public SourceImplementationsHandler(ConfigPersistence configPersistence,
|
||||
IntegrationSchemaValidation integrationSchemaValidation) {
|
||||
this(configPersistence, integrationSchemaValidation, UUID::randomUUID);
|
||||
}
|
||||
|
||||
public SourceImplementationRead createSourceImplementation(
|
||||
SourceImplementationCreate sourceImplementationCreate) {
|
||||
public SourceImplementationRead createSourceImplementation(SourceImplementationCreate sourceImplementationCreate) {
|
||||
// validate configuration
|
||||
validateSourceImplementation(
|
||||
sourceImplementationCreate.getSourceSpecificationId(),
|
||||
@@ -83,8 +80,7 @@ public class SourceImplementationsHandler {
|
||||
return getSourceImplementationReadInternal(sourceImplementationId);
|
||||
}
|
||||
|
||||
public SourceImplementationRead updateSourceImplementation(
|
||||
SourceImplementationUpdate sourceImplementationUpdate) {
|
||||
public SourceImplementationRead updateSourceImplementation(SourceImplementationUpdate sourceImplementationUpdate) {
|
||||
// get existing implementation
|
||||
final SourceConnectionImplementation persistedSourceImplementation =
|
||||
getSourceConnectionImplementationInternal(
|
||||
@@ -108,21 +104,18 @@ public class SourceImplementationsHandler {
|
||||
sourceImplementationUpdate.getSourceImplementationId());
|
||||
}
|
||||
|
||||
public SourceImplementationRead getSourceImplementation(
|
||||
SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public SourceImplementationRead getSourceImplementation(SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
|
||||
return getSourceImplementationReadInternal(
|
||||
sourceImplementationIdRequestBody.getSourceImplementationId());
|
||||
}
|
||||
|
||||
public SourceImplementationReadList listSourceImplementationsForWorkspace(
|
||||
WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
public SourceImplementationReadList listSourceImplementationsForWorkspace(WorkspaceIdRequestBody workspaceIdRequestBody) {
|
||||
|
||||
final List<SourceImplementationRead> reads =
|
||||
ConfigFetchers.getSourceConnectionImplementations(configPersistence).stream()
|
||||
.filter(
|
||||
sourceConnectionImplementation ->
|
||||
sourceConnectionImplementation
|
||||
sourceConnectionImplementation -> sourceConnectionImplementation
|
||||
.getWorkspaceId()
|
||||
.equals(workspaceIdRequestBody.getWorkspaceId()))
|
||||
.map(
|
||||
@@ -142,8 +135,7 @@ public class SourceImplementationsHandler {
|
||||
return sourceImplementationReadList;
|
||||
}
|
||||
|
||||
public void deleteSourceImplementation(
|
||||
SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
public void deleteSourceImplementation(SourceImplementationIdRequestBody sourceImplementationIdRequestBody) {
|
||||
// get existing implementation
|
||||
final SourceImplementationRead persistedSourceImplementation =
|
||||
getSourceImplementationReadInternal(
|
||||
@@ -158,14 +150,12 @@ public class SourceImplementationsHandler {
|
||||
(String) persistedSourceImplementation.getConnectionConfiguration());
|
||||
}
|
||||
|
||||
private SourceConnectionImplementation getSourceConnectionImplementationInternal(
|
||||
UUID sourceImplementationId) {
|
||||
private SourceConnectionImplementation getSourceConnectionImplementationInternal(UUID sourceImplementationId) {
|
||||
return ConfigFetchers.getSourceConnectionImplementation(
|
||||
configPersistence, sourceImplementationId);
|
||||
}
|
||||
|
||||
private SourceImplementationRead getSourceImplementationReadInternal(
|
||||
UUID sourceImplementationId) {
|
||||
private SourceImplementationRead getSourceImplementationReadInternal(UUID sourceImplementationId) {
|
||||
// read configuration from db
|
||||
final SourceConnectionImplementation retrievedSourceConnectionImplementation =
|
||||
getSourceConnectionImplementationInternal(sourceImplementationId);
|
||||
@@ -179,8 +169,7 @@ public class SourceImplementationsHandler {
|
||||
return toSourceImplementationRead(retrievedSourceConnectionImplementation, sourceId);
|
||||
}
|
||||
|
||||
private void validateSourceImplementation(
|
||||
UUID sourceConnectionSpecificationId, String implementationJson) {
|
||||
private void validateSourceImplementation(UUID sourceConnectionSpecificationId, String implementationJson) {
|
||||
try {
|
||||
validator.validateSourceConnectionConfiguration(
|
||||
sourceConnectionSpecificationId, implementationJson);
|
||||
@@ -193,8 +182,7 @@ public class SourceImplementationsHandler {
|
||||
}
|
||||
}
|
||||
|
||||
private void persistSourceConnectionImplementation(
|
||||
UUID sourceSpecificationId,
|
||||
private void persistSourceConnectionImplementation(UUID sourceSpecificationId,
|
||||
UUID workspaceId,
|
||||
UUID sourceImplementationId,
|
||||
boolean tombstone,
|
||||
@@ -214,8 +202,7 @@ public class SourceImplementationsHandler {
|
||||
sourceConnectionImplementation);
|
||||
}
|
||||
|
||||
private SourceImplementationRead toSourceImplementationRead(
|
||||
SourceConnectionImplementation sourceConnectionImplementation, UUID sourceId) {
|
||||
private SourceImplementationRead toSourceImplementationRead(SourceConnectionImplementation sourceConnectionImplementation, UUID sourceId) {
|
||||
final SourceImplementationRead sourceImplementationRead = new SourceImplementationRead();
|
||||
sourceImplementationRead.setSourceId(sourceId);
|
||||
sourceImplementationRead.setSourceImplementationId(
|
||||
@@ -228,4 +215,5 @@ public class SourceImplementationsHandler {
|
||||
|
||||
return sourceImplementationRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -47,12 +47,10 @@ public class SourceSpecificationsHandler {
|
||||
sourceConnection =
|
||||
ConfigFetchers.getSourceConnectionSpecifications(configPersistence).stream()
|
||||
.filter(
|
||||
sourceSpecification ->
|
||||
sourceSpecification.getSourceId().equals(sourceIdRequestBody.getSourceId()))
|
||||
sourceSpecification -> sourceSpecification.getSourceId().equals(sourceIdRequestBody.getSourceId()))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() ->
|
||||
new KnownException(
|
||||
() -> new KnownException(
|
||||
404,
|
||||
String.format(
|
||||
"Could not find a source specification for source: %s",
|
||||
@@ -61,8 +59,7 @@ public class SourceSpecificationsHandler {
|
||||
return toSourceSpecificationRead(sourceConnection);
|
||||
}
|
||||
|
||||
private static SourceSpecificationRead toSourceSpecificationRead(
|
||||
SourceConnectionSpecification sourceConnectionSpecification) {
|
||||
private static SourceSpecificationRead toSourceSpecificationRead(SourceConnectionSpecification sourceConnectionSpecification) {
|
||||
final SourceSpecificationRead sourceSpecificationRead = new SourceSpecificationRead();
|
||||
sourceSpecificationRead.setSourceId(sourceConnectionSpecification.getSourceId());
|
||||
sourceSpecificationRead.setSourceSpecificationId(
|
||||
@@ -72,4 +69,5 @@ public class SourceSpecificationsHandler {
|
||||
|
||||
return sourceSpecificationRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SourcesHandler {
|
||||
|
||||
private final ConfigPersistence configPersistence;
|
||||
|
||||
public SourcesHandler(ConfigPersistence configPersistence) {
|
||||
@@ -67,4 +68,5 @@ public class SourcesHandler {
|
||||
|
||||
return sourceRead;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ import io.dataline.server.helpers.ConfigFetchers;
|
||||
import java.util.UUID;
|
||||
|
||||
public class WorkspacesHandler {
|
||||
|
||||
private final ConfigPersistence configPersistence;
|
||||
|
||||
public WorkspacesHandler(ConfigPersistence configPersistence) {
|
||||
@@ -86,4 +87,5 @@ public class WorkspacesHandler {
|
||||
|
||||
return getWorkspaceFromId(workspaceUpdate.getWorkspaceId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -48,8 +48,8 @@ import java.util.UUID;
|
||||
*/
|
||||
public class ConfigFetchers {
|
||||
|
||||
public static StandardWorkspace getStandardWorkspace(
|
||||
ConfigPersistence configPersistence, UUID workspaceId) {
|
||||
public static StandardWorkspace getStandardWorkspace(ConfigPersistence configPersistence,
|
||||
UUID workspaceId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_WORKSPACE,
|
||||
@@ -62,8 +62,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardSource getStandardSource(
|
||||
ConfigPersistence configPersistence, UUID sourceId) {
|
||||
public static StandardSource getStandardSource(ConfigPersistence configPersistence,
|
||||
UUID sourceId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_SOURCE, sourceId.toString(), StandardSource.class);
|
||||
@@ -75,8 +75,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
|
||||
// wrap json validation errors for usages in API handlers.
|
||||
public static <T> void writeConfig(
|
||||
ConfigPersistence configPersistence,
|
||||
public static <T> void writeConfig(ConfigPersistence configPersistence,
|
||||
PersistenceConfigType persistenceConfigType,
|
||||
String configId,
|
||||
T config) {
|
||||
@@ -96,8 +95,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static SourceConnectionSpecification getSourceConnectionSpecification(
|
||||
ConfigPersistence configPersistence, UUID sourceSpecificationId) {
|
||||
public static SourceConnectionSpecification getSourceConnectionSpecification(ConfigPersistence configPersistence,
|
||||
UUID sourceSpecificationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.SOURCE_CONNECTION_SPECIFICATION,
|
||||
@@ -110,8 +109,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<SourceConnectionSpecification> getSourceConnectionSpecifications(
|
||||
ConfigPersistence configPersistence) {
|
||||
public static Set<SourceConnectionSpecification> getSourceConnectionSpecifications(ConfigPersistence configPersistence) {
|
||||
try {
|
||||
return configPersistence.getConfigs(
|
||||
PersistenceConfigType.SOURCE_CONNECTION_SPECIFICATION,
|
||||
@@ -121,8 +119,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static SourceConnectionImplementation getSourceConnectionImplementation(
|
||||
ConfigPersistence configPersistence, UUID sourceImplementationId) {
|
||||
public static SourceConnectionImplementation getSourceConnectionImplementation(ConfigPersistence configPersistence,
|
||||
UUID sourceImplementationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.SOURCE_CONNECTION_IMPLEMENTATION,
|
||||
@@ -135,8 +133,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<SourceConnectionImplementation> getSourceConnectionImplementations(
|
||||
ConfigPersistence configPersistence) {
|
||||
public static Set<SourceConnectionImplementation> getSourceConnectionImplementations(ConfigPersistence configPersistence) {
|
||||
try {
|
||||
return configPersistence.getConfigs(
|
||||
PersistenceConfigType.SOURCE_CONNECTION_IMPLEMENTATION,
|
||||
@@ -146,8 +143,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardDestination getStandardDestination(
|
||||
ConfigPersistence configPersistence, UUID destinationId) {
|
||||
public static StandardDestination getStandardDestination(ConfigPersistence configPersistence,
|
||||
UUID destinationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_DESTINATION,
|
||||
@@ -160,8 +157,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<StandardDestination> getStandardDestinations(
|
||||
ConfigPersistence configPersistence) {
|
||||
public static Set<StandardDestination> getStandardDestinations(ConfigPersistence configPersistence) {
|
||||
try {
|
||||
return configPersistence.getConfigs(
|
||||
PersistenceConfigType.STANDARD_DESTINATION, StandardDestination.class);
|
||||
@@ -170,8 +166,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static DestinationConnectionSpecification getDestinationConnectionSpecification(
|
||||
ConfigPersistence configPersistence, UUID destinationSpecificationId) {
|
||||
public static DestinationConnectionSpecification getDestinationConnectionSpecification(ConfigPersistence configPersistence,
|
||||
UUID destinationSpecificationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.DESTINATION_CONNECTION_SPECIFICATION,
|
||||
@@ -185,8 +181,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<DestinationConnectionSpecification> getDestinationConnectionSpecifications(
|
||||
ConfigPersistence configPersistence) {
|
||||
public static Set<DestinationConnectionSpecification> getDestinationConnectionSpecifications(ConfigPersistence configPersistence) {
|
||||
try {
|
||||
return configPersistence.getConfigs(
|
||||
PersistenceConfigType.DESTINATION_CONNECTION_SPECIFICATION,
|
||||
@@ -196,8 +191,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static DestinationConnectionImplementation getDestinationConnectionImplementation(
|
||||
ConfigPersistence configPersistence, UUID destinationImplementationId) {
|
||||
public static DestinationConnectionImplementation getDestinationConnectionImplementation(ConfigPersistence configPersistence,
|
||||
UUID destinationImplementationId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.DESTINATION_CONNECTION_IMPLEMENTATION,
|
||||
@@ -211,8 +206,7 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static Set<DestinationConnectionImplementation> getDestinationConnectionImplementations(
|
||||
ConfigPersistence configPersistence) {
|
||||
public static Set<DestinationConnectionImplementation> getDestinationConnectionImplementations(ConfigPersistence configPersistence) {
|
||||
try {
|
||||
return configPersistence.getConfigs(
|
||||
PersistenceConfigType.DESTINATION_CONNECTION_IMPLEMENTATION,
|
||||
@@ -222,8 +216,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardSync getStandardSync(
|
||||
ConfigPersistence configPersistence, UUID connectionId) {
|
||||
public static StandardSync getStandardSync(ConfigPersistence configPersistence,
|
||||
UUID connectionId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_SYNC, connectionId.toString(), StandardSync.class);
|
||||
@@ -242,8 +236,8 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardSyncSchedule getStandardSyncSchedule(
|
||||
ConfigPersistence configPersistence, UUID connectionId) {
|
||||
public static StandardSyncSchedule getStandardSyncSchedule(ConfigPersistence configPersistence,
|
||||
UUID connectionId) {
|
||||
try {
|
||||
return configPersistence.getConfig(
|
||||
PersistenceConfigType.STANDARD_SYNC_SCHEDULE,
|
||||
@@ -256,8 +250,9 @@ public class ConfigFetchers {
|
||||
}
|
||||
}
|
||||
|
||||
private static KnownException getConfigNotFoundException(
|
||||
Throwable e, String configName, UUID id) {
|
||||
private static KnownException getConfigNotFoundException(Throwable e,
|
||||
String configName,
|
||||
UUID id) {
|
||||
return new KnownException(
|
||||
422, String.format("Could not find sync configuration for %s: %s.", configName, id), e);
|
||||
}
|
||||
@@ -270,4 +265,5 @@ public class ConfigFetchers {
|
||||
e.getMessage()),
|
||||
e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -46,8 +46,7 @@ public class IntegrationSchemaValidation {
|
||||
this.jsonSchemaValidation = new JsonSchemaValidation();
|
||||
}
|
||||
|
||||
public void validateSourceConnectionConfiguration(
|
||||
UUID sourceConnectionSpecificationId, String configurationJson)
|
||||
public void validateSourceConnectionConfiguration(UUID sourceConnectionSpecificationId, String configurationJson)
|
||||
throws JsonValidationException {
|
||||
final SourceConnectionSpecification sourceConnectionSpecification =
|
||||
ConfigFetchers.getSourceConnectionSpecification(
|
||||
@@ -60,8 +59,7 @@ public class IntegrationSchemaValidation {
|
||||
jsonSchemaValidation.validateThrow(schemaJson, configJson);
|
||||
}
|
||||
|
||||
public void validateDestinationConnectionConfiguration(
|
||||
UUID destinationConnectionSpecificationId, String configurationJson)
|
||||
public void validateDestinationConnectionConfiguration(UUID destinationConnectionSpecificationId, String configurationJson)
|
||||
throws JsonValidationException {
|
||||
final DestinationConnectionSpecification destinationConnectionSpecification =
|
||||
ConfigFetchers.getDestinationConnectionSpecification(
|
||||
@@ -73,4 +71,5 @@ public class IntegrationSchemaValidation {
|
||||
|
||||
jsonSchemaValidation.validateThrow(schemaJson, configJson);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,8 +29,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ServerAppTest {
|
||||
|
||||
@Test
|
||||
void name() {
|
||||
assertEquals(1, 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -326,8 +326,9 @@ class ConnectionsHandlerTest {
|
||||
return connectionSchedule;
|
||||
}
|
||||
|
||||
private ConnectionRead generateExpectedConnectionRead(
|
||||
UUID connectionId, UUID sourceImplementationId, UUID destinationImplementationId) {
|
||||
private ConnectionRead generateExpectedConnectionRead(UUID connectionId,
|
||||
UUID sourceImplementationId,
|
||||
UUID destinationImplementationId) {
|
||||
final ConnectionRead expectedConnectionRead = new ConnectionRead();
|
||||
expectedConnectionRead.setConnectionId(connectionId);
|
||||
expectedConnectionRead.setSourceImplementationId(sourceImplementationId);
|
||||
@@ -360,4 +361,5 @@ class ConnectionsHandlerTest {
|
||||
|
||||
return standardSchedule;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class DestinationImplementationsHandlerTest {
|
||||
|
||||
private ConfigPersistence configPersistence;
|
||||
private DestinationConnectionSpecification destinationConnectionSpecification;
|
||||
private DestinationConnectionImplementation destinationConnectionImplementation;
|
||||
@@ -88,8 +89,8 @@ class DestinationImplementationsHandlerTest {
|
||||
return Files.readString(path);
|
||||
}
|
||||
|
||||
private DestinationConnectionImplementation generateDestinationImplementation(
|
||||
UUID destinationSpecificationId) throws IOException {
|
||||
private DestinationConnectionImplementation generateDestinationImplementation(UUID destinationSpecificationId)
|
||||
throws IOException {
|
||||
final UUID workspaceId = UUID.randomUUID();
|
||||
final UUID destinationImplementationId = UUID.randomUUID();
|
||||
|
||||
@@ -301,4 +302,5 @@ class DestinationImplementationsHandlerTest {
|
||||
expectedDestinationImplementationRead,
|
||||
actualDestinationImplementationRead.getDestinations().get(0));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -79,4 +79,5 @@ class DestinationSpecificationsHandlerTest {
|
||||
|
||||
assertEquals(expectedDestinationSpecificationRead, actualDestinationSpecificationRead);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class DestinationsHandlerTest {
|
||||
|
||||
private ConfigPersistence configPersistence;
|
||||
private StandardDestination destination;
|
||||
private DestinationsHandler destinationHandler;
|
||||
@@ -90,14 +91,12 @@ class DestinationsHandlerTest {
|
||||
final Optional<DestinationRead> actualDestinationRead1 =
|
||||
actualDestinationReadList.getDestinations().stream()
|
||||
.filter(
|
||||
destinationRead ->
|
||||
destinationRead.getDestinationId().equals(destination.getDestinationId()))
|
||||
destinationRead -> destinationRead.getDestinationId().equals(destination.getDestinationId()))
|
||||
.findFirst();
|
||||
final Optional<DestinationRead> actualDestinationRead2 =
|
||||
actualDestinationReadList.getDestinations().stream()
|
||||
.filter(
|
||||
destinationRead ->
|
||||
destinationRead.getDestinationId().equals(destination2.getDestinationId()))
|
||||
destinationRead -> destinationRead.getDestinationId().equals(destination2.getDestinationId()))
|
||||
.findFirst();
|
||||
|
||||
assertTrue(actualDestinationRead1.isPresent());
|
||||
@@ -126,4 +125,5 @@ class DestinationsHandlerTest {
|
||||
|
||||
assertEquals(expectedDestinationRead, actualDestinationRead);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SourceImplementationsHandlerTest {
|
||||
|
||||
private ConfigPersistence configPersistence;
|
||||
private SourceConnectionSpecification sourceConnectionSpecification;
|
||||
private SourceConnectionImplementation sourceConnectionImplementation;
|
||||
@@ -311,4 +312,5 @@ class SourceImplementationsHandlerTest {
|
||||
sourceConnectionImplementation.getSourceImplementationId().toString(),
|
||||
expectedSourceConnectionImplementation);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -75,4 +75,5 @@ class SourceSpecificationsHandlerTest {
|
||||
|
||||
assertEquals(expectedSourceSpecificationRead, actualSourceSpecificationRead);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SourcesHandlerTest {
|
||||
|
||||
private ConfigPersistence configPersistence;
|
||||
private StandardSource source;
|
||||
private SourcesHandler sourceHandler;
|
||||
@@ -118,4 +119,5 @@ class SourcesHandlerTest {
|
||||
|
||||
assertEquals(expectedSourceRead, actualSourceRead);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class WorkspacesHandlerTest {
|
||||
|
||||
private ConfigPersistence configPersistence;
|
||||
private StandardWorkspace workspace;
|
||||
private WorkspacesHandler workspacesHandler;
|
||||
@@ -151,4 +152,5 @@ class WorkspacesHandlerTest {
|
||||
|
||||
assertEquals(expectedWorkspaceRead, actualWorkspaceRead);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -49,4 +49,5 @@ public class DestinationSpecificationHelpers {
|
||||
|
||||
return destinationConnectionSpecification;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -33,8 +33,8 @@ import java.util.UUID;
|
||||
|
||||
public class SourceImplementationHelpers {
|
||||
|
||||
public static SourceConnectionImplementation generateSourceImplementation(
|
||||
UUID sourceSpecificationId) throws IOException {
|
||||
public static SourceConnectionImplementation generateSourceImplementation(UUID sourceSpecificationId)
|
||||
throws IOException {
|
||||
final UUID workspaceId = UUID.randomUUID();
|
||||
final UUID sourceImplementationId = UUID.randomUUID();
|
||||
|
||||
@@ -56,4 +56,5 @@ public class SourceImplementationHelpers {
|
||||
Paths.get("../dataline-server/src/test/resources/json/TestImplementation.json");
|
||||
return Files.readString(path);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -48,4 +48,5 @@ public class SourceSpecificationHelpers {
|
||||
|
||||
return sourceConnectionSpecification;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DefaultSyncWorker implements SyncWorker {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSyncWorker.class);
|
||||
|
||||
public static final String TAP_ERR_LOG = "tap_err.log";
|
||||
@@ -49,8 +50,7 @@ public class DefaultSyncWorker implements SyncWorker {
|
||||
|
||||
private final AtomicBoolean cancelled;
|
||||
|
||||
public DefaultSyncWorker(
|
||||
TapFactory<SingerMessage> singerTapFactory,
|
||||
public DefaultSyncWorker(TapFactory<SingerMessage> singerTapFactory,
|
||||
TargetFactory<SingerMessage> singerTargetFactory) {
|
||||
this.singerTapFactory = singerTapFactory;
|
||||
this.singerTargetFactory = singerTargetFactory;
|
||||
@@ -98,4 +98,5 @@ public class DefaultSyncWorker implements SyncWorker {
|
||||
public void cancel() {
|
||||
cancelled.set(true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ package io.dataline.workers;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
public class DiscoveryOutput {
|
||||
|
||||
// TODO line this up with conduit config type
|
||||
private final String catalog;
|
||||
|
||||
@@ -42,4 +43,5 @@ public class DiscoveryOutput {
|
||||
public String toString() {
|
||||
return new ToStringBuilder(this).append(catalog).toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class EchoWorker implements Worker<String, String> {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(EchoWorker.class);
|
||||
|
||||
public EchoWorker() {}
|
||||
@@ -43,4 +44,5 @@ public class EchoWorker implements Worker<String, String> {
|
||||
public void cancel() {
|
||||
// no-op
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@
|
||||
package io.dataline.workers;
|
||||
|
||||
public class InvalidCatalogException extends Exception {
|
||||
|
||||
public InvalidCatalogException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@
|
||||
package io.dataline.workers;
|
||||
|
||||
public class InvalidCredentialsException extends Exception {
|
||||
|
||||
public InvalidCredentialsException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import java.util.Optional;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
public class OutputAndStatus<OutputType> {
|
||||
|
||||
private final OutputType output;
|
||||
private final JobStatus status;
|
||||
|
||||
@@ -53,4 +54,5 @@ public class OutputAndStatus<OutputType> {
|
||||
public JobStatus getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ import java.nio.file.Path;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public interface TapFactory<T> {
|
||||
|
||||
Stream<T> create(StandardTapConfig tapConfig, Path workspacePath)
|
||||
throws InvalidCredentialsException, InvalidCatalogException, SyncException;
|
||||
|
||||
}
|
||||
|
||||
@@ -60,4 +60,5 @@ public class TargetConsumer implements CloseableConsumer<SingerMessage> {
|
||||
writer.flush();
|
||||
WorkerUtils.closeProcess(process);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ import io.dataline.config.StandardTargetConfig;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public interface TargetFactory<T> {
|
||||
|
||||
CloseableConsumer<T> create(StandardTargetConfig targetConfig, Path workspacePath)
|
||||
throws SyncException;
|
||||
|
||||
}
|
||||
|
||||
@@ -27,12 +27,14 @@ package io.dataline.workers;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public interface Worker<InputType, OutputType> {
|
||||
|
||||
/**
|
||||
* Blocking call to run the worker's workflow. Once this is complete, getStatus should return
|
||||
* either COMPLETE, FAILED, or CANCELLED.
|
||||
* Blocking call to run the worker's workflow. Once this is complete, getStatus should return either
|
||||
* COMPLETE, FAILED, or CANCELLED.
|
||||
*/
|
||||
OutputAndStatus<OutputType> run(InputType inputType, Path jobRoot)
|
||||
throws InvalidCredentialsException, InvalidCatalogException;
|
||||
|
||||
void cancel();
|
||||
|
||||
}
|
||||
|
||||
@@ -105,4 +105,5 @@ public class WorkerUtils {
|
||||
targetConfig.setStandardSync(sync.getStandardSync());
|
||||
return targetConfig;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -71,4 +71,5 @@ public class DockerProcessBuilderFactory implements ProcessBuilderFactory {
|
||||
final Path relativePath = workspaceRoot.relativize(jobRoot);
|
||||
return MOUNT_DESTINATION.resolve(relativePath);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -29,4 +29,5 @@ import java.nio.file.Path;
|
||||
public interface ProcessBuilderFactory {
|
||||
|
||||
ProcessBuilder create(Path jobPath, String imageName, String... args);
|
||||
|
||||
}
|
||||
|
||||
@@ -39,8 +39,9 @@ public class MessageUtils {
|
||||
private static final DateTimeFormatter SINGER_DATETIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneId.of("UTC"));
|
||||
|
||||
public static SingerMessage createRecordMessage(
|
||||
String tableName, JsonNode record, Instant timeExtracted) {
|
||||
public static SingerMessage createRecordMessage(String tableName,
|
||||
JsonNode record,
|
||||
Instant timeExtracted) {
|
||||
final SingerMessage singerMessage = new SingerMessage();
|
||||
singerMessage.setType(SingerMessage.Type.RECORD);
|
||||
singerMessage.setRecord(Jsons.serialize(record));
|
||||
@@ -60,4 +61,5 @@ public class MessageUtils {
|
||||
public static SingerMessage createRecordMessage(String tableName, Map<String, String> record) {
|
||||
return createRecordMessage(tableName, Jsons.jsonNode(record), null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -38,9 +38,10 @@ import org.slf4j.LoggerFactory;
|
||||
* InputStream into a SingerMessage. If the line cannot be parsed into a SingerMessage it is
|
||||
* dropped. Each record MUST be new line separated.
|
||||
*
|
||||
* <p>If a line starts with a SingerMessage and then has other characters after it, that
|
||||
* SingerMessage will still be parsed. If there are multiple SingerMessage records on the same line,
|
||||
* only the first will be parsed.
|
||||
* <p>
|
||||
* If a line starts with a SingerMessage and then has other characters after it, that SingerMessage
|
||||
* will still be parsed. If there are multiple SingerMessage records on the same line, only the
|
||||
* first will be parsed.
|
||||
*/
|
||||
public class SingerJsonStreamFactory {
|
||||
|
||||
@@ -59,4 +60,5 @@ public class SingerJsonStreamFactory {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -65,4 +65,5 @@ public class SingerMessageTracker implements Consumer<SingerMessage> {
|
||||
public Optional<State> getOutputState() {
|
||||
return Optional.ofNullable(outputState.get());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -72,4 +72,5 @@ public abstract class BaseSingerWorker<InputType, OutputType>
|
||||
protected static Path getFullPath(Path workspaceRoot, String fileName) {
|
||||
return workspaceRoot.resolve(fileName);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -43,8 +43,8 @@ public class SingerCatalogConverters {
|
||||
|
||||
/**
|
||||
* Takes in a singer catalog and a dataline schema. It then applies the dataline configuration to
|
||||
* that catalog. e.g. If dataline says that a certain column should or should not be included in
|
||||
* the sync, this method applies that to the catalog. Thus we produce a valid singer catalog that
|
||||
* that catalog. e.g. If dataline says that a certain column should or should not be included in the
|
||||
* sync, this method applies that to the catalog. Thus we produce a valid singer catalog that
|
||||
* contains configurations stored in dataline.
|
||||
*
|
||||
* @param catalog - singer catalog
|
||||
@@ -189,15 +189,14 @@ public class SingerCatalogConverters {
|
||||
return false;
|
||||
}
|
||||
|
||||
private static Map<String, List<SingerMetadata>> getTableNameToMetadataList(
|
||||
List<SingerStream> streams) {
|
||||
private static Map<String, List<SingerMetadata>> getTableNameToMetadataList(List<SingerStream> streams) {
|
||||
// todo (cgardens) - figure out if it's stream or stream id or table name.
|
||||
return streams.stream()
|
||||
.collect(Collectors.toMap(SingerStream::getStream, SingerStream::getMetadata));
|
||||
}
|
||||
|
||||
private static Map<String, SingerMetadataChild> getColumnMetadataForTable(
|
||||
Map<String, List<SingerMetadata>> tableNameToMetadata, String tableName) {
|
||||
private static Map<String, SingerMetadataChild> getColumnMetadataForTable(Map<String, List<SingerMetadata>> tableNameToMetadata,
|
||||
String tableName) {
|
||||
if (!tableNameToMetadata.containsKey(tableName)) {
|
||||
throw new RuntimeException("could not find metadata for table: " + tableName);
|
||||
}
|
||||
@@ -229,8 +228,8 @@ public class SingerCatalogConverters {
|
||||
}
|
||||
|
||||
/**
|
||||
* Singer tends to have 2 types for columns one of which is null. The null is pretty irrelevant,
|
||||
* so look at types and find the first non-null one and use that.
|
||||
* Singer tends to have 2 types for columns one of which is null. The null is pretty irrelevant, so
|
||||
* look at types and find the first non-null one and use that.
|
||||
*
|
||||
* @param singerTypes - list of types discovered by singer.
|
||||
* @return reduce down to one type which best matches the column's data type
|
||||
@@ -244,9 +243,9 @@ public class SingerCatalogConverters {
|
||||
}
|
||||
|
||||
/**
|
||||
* Singer doesn't seem to have an official list of the data types that they support, so we will
|
||||
* have to do our best here as we discover them. If it becomes too awful, we can just map types we
|
||||
* don't recognize to string.
|
||||
* Singer doesn't seem to have an official list of the data types that they support, so we will have
|
||||
* to do our best here as we discover them. If it becomes too awful, we can just map types we don't
|
||||
* recognize to string.
|
||||
*
|
||||
* @param singerType - singer's column data type
|
||||
* @return best match for our own data type
|
||||
@@ -293,4 +292,5 @@ public class SingerCatalogConverters {
|
||||
|
||||
return singerMetadata;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -50,8 +50,9 @@ public class SingerCheckConnectionWorker
|
||||
}
|
||||
|
||||
@Override
|
||||
public OutputAndStatus<StandardCheckConnectionOutput> run(
|
||||
StandardCheckConnectionInput input, Path jobRoot) throws InvalidCredentialsException {
|
||||
public OutputAndStatus<StandardCheckConnectionOutput> run(StandardCheckConnectionInput input,
|
||||
Path jobRoot)
|
||||
throws InvalidCredentialsException {
|
||||
|
||||
final StandardDiscoverSchemaInput discoverSchemaInput = new StandardDiscoverSchemaInput();
|
||||
discoverSchemaInput.setConnectionConfigurationJson(input.getConnectionConfigurationJson());
|
||||
@@ -79,4 +80,5 @@ public class SingerCheckConnectionWorker
|
||||
public void cancel() {
|
||||
singerDiscoverSchemaWorker.cancel();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -66,8 +66,8 @@ public class SingerDiscoverSchemaWorker
|
||||
|
||||
// package private since package-local classes need direct access to singer catalog, and the
|
||||
// conversion from SingerSchema to Dataline schema is lossy
|
||||
OutputAndStatus<SingerCatalog> runInternal(
|
||||
StandardDiscoverSchemaInput discoverSchemaInput, Path jobRoot)
|
||||
OutputAndStatus<SingerCatalog> runInternal(StandardDiscoverSchemaInput discoverSchemaInput,
|
||||
Path jobRoot)
|
||||
throws InvalidCredentialsException {
|
||||
// todo (cgardens) - just getting original impl to line up with new iface for now. this can be
|
||||
// reduced.
|
||||
@@ -105,8 +105,8 @@ public class SingerDiscoverSchemaWorker
|
||||
}
|
||||
|
||||
@Override
|
||||
public OutputAndStatus<StandardDiscoverSchemaOutput> run(
|
||||
StandardDiscoverSchemaInput discoverSchemaInput, Path jobRoot)
|
||||
public OutputAndStatus<StandardDiscoverSchemaOutput> run(StandardDiscoverSchemaInput discoverSchemaInput,
|
||||
Path jobRoot)
|
||||
throws InvalidCredentialsException {
|
||||
OutputAndStatus<SingerCatalog> output = runInternal(discoverSchemaInput, jobRoot);
|
||||
JobStatus status = output.getStatus();
|
||||
@@ -131,4 +131,5 @@ public class SingerDiscoverSchemaWorker
|
||||
public void cancel() {
|
||||
cancelHelper(workerProcess);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -126,4 +126,5 @@ public class SingerTapFactory implements TapFactory<SingerMessage> {
|
||||
return new SingerDiscoverSchemaWorker(imageName, pbf)
|
||||
.runInternal(discoveryInput, scopedWorkspace);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -81,4 +81,5 @@ public class SingerTargetFactory implements TargetFactory<SingerMessage> {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ import org.junit.jupiter.api.TestInstance;
|
||||
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public abstract class BaseWorkerTestCase {
|
||||
|
||||
// TODO inject via env
|
||||
protected Path workspaceRoot;
|
||||
protected ProcessBuilderFactory pbf;
|
||||
@@ -71,4 +72,5 @@ public abstract class BaseWorkerTestCase {
|
||||
protected void assertJsonEquals(final String s1, final String s2) {
|
||||
assertEquals(Jsons.deserialize(s1), Jsons.deserialize(s2));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class DefaultSyncWorkerTest extends BaseWorkerTestCase {
|
||||
|
||||
private static final Path WORKSPACE_ROOT = Path.of("/workspaces/10");
|
||||
private static final String TABLE_NAME = "user_preferences";
|
||||
private static final String COLUMN_NAME = "favorite_color";
|
||||
@@ -168,4 +169,5 @@ class DefaultSyncWorkerTest extends BaseWorkerTestCase {
|
||||
verify(consumer).accept(recordMessage2);
|
||||
verify(consumer).close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user